Tor  0.4.7.0-alpha-dev
rephist.c
Go to the documentation of this file.
1 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
2  * Copyright (c) 2007-2021, The Tor Project, Inc. */
3 /* See LICENSE for licensing information */
4 
5 /**
6  * \file rephist.c
7  * \brief Basic history and performance-tracking functionality.
8  *
9  * Basic history and performance-tracking functionality to remember
10  * which servers have worked in the past, how much bandwidth we've
11  * been using, which ports we tend to want, and so on; further,
12  * exit port statistics, cell statistics, and connection statistics.
13  *
14  * The history and information tracked in this module could sensibly be
15  * divided into several categories:
16  *
17  * <ul><li>Statistics used by authorities to remember the uptime and
18  * stability information about various relays, including "uptime",
19  * "weighted fractional uptime" and "mean time between failures".
20  *
21  * <li>Predicted ports, used by clients to remember how long it's been
22  * since they opened an exit connection to each given target
23  * port. Clients use this information in order to try to keep circuits
24  * open to exit nodes that can connect to the ports that they care
25  * about. (The predicted ports mechanism also handles predicted circuit
26  * usage that _isn't_ port-specific, such as resolves, internal circuits,
27  * and so on.)
28  *
29  * <li>Public key operation counters, for tracking how many times we've
30  * done each public key operation. (This is unmaintained and we should
31  * remove it.)
32  *
33  * <li>Exit statistics by port, used by exits to keep track of the
34  * number of streams and bytes they've served at each exit port, so they
35  * can generate their exit-kibibytes-{read,written} and
36  * exit-streams-opened statistics.
37  *
38  * <li>Circuit stats, used by relays instances to tract circuit
39  * queue fullness and delay over time, and generate cell-processed-cells,
40  * cell-queued-cells, cell-time-in-queue, and cell-circuits-per-decile
41  * statistics.
42  *
43  * <li>Descriptor serving statistics, used by directory caches to track
44  * how many descriptors they've served.
45  *
46  * <li>Onion handshake statistics, used by relays to count how many
47  * TAP and ntor handshakes they've handled.
48  *
49  * <li>Hidden service statistics, used by relays to count rendezvous
50  * traffic and HSDir-stored descriptors.
51  *
52  * <li>Link protocol statistics, used by relays to count how many times
53  * each link protocol has been used.
54  *
55  * </ul>
56  *
57  * The entry points for this module are scattered throughout the
58  * codebase. Sending data, receiving data, connecting to a relay,
59  * losing a connection to a relay, and so on can all trigger a change in
60  * our current stats. Relays also invoke this module in order to
61  * extract their statistics when building routerinfo and extrainfo
62  * objects in router.c.
63  *
64  * TODO: This module should be broken up.
65  *
66  * (The "rephist" name originally stood for "reputation and history". )
67  **/
68 
69 #define REPHIST_PRIVATE
70 #include "core/or/or.h"
71 #include "app/config/config.h"
72 #include "core/or/circuitlist.h"
73 #include "core/or/connection_or.h"
79 #include "feature/stats/rephist.h"
80 #include "lib/container/order.h"
82 #include "lib/math/laplace.h"
83 
85 #include "core/or/or_circuit_st.h"
86 
87 #ifdef HAVE_FCNTL_H
88 #include <fcntl.h>
89 #endif
90 
91 /** Total number of bytes currently allocated in fields used by rephist.c. */
93 /** Number of or_history_t objects currently allocated. */
94 uint32_t rephist_total_num=0;
95 
96 /** If the total weighted run count of all runs for a router ever falls
97  * below this amount, the router can be treated as having 0 MTBF. */
98 #define STABILITY_EPSILON 0.0001
99 /** Value by which to discount all old intervals for MTBF purposes. This
100  * is compounded every STABILITY_INTERVAL. */
101 #define STABILITY_ALPHA 0.95
102 /** Interval at which to discount all old intervals for MTBF purposes. */
103 #define STABILITY_INTERVAL (12*60*60)
104 /* (This combination of ALPHA, INTERVAL, and EPSILON makes it so that an
105  * interval that just ended counts twice as much as one that ended a week ago,
106  * 20X as much as one that ended a month ago, and routers that have had no
107  * uptime data for about half a year will get forgotten.) */
108 
109 /** History of an OR. */
110 typedef struct or_history_t {
111  /** When did we start tracking this OR? */
112  time_t since;
113  /** When did we most recently note a change to this OR? */
114  time_t changed;
115 
116  /** The address at which we most recently connected to this OR
117  * successfully. */
119 
120  /** The port at which we most recently connected to this OR successfully */
122 
123  /* === For MTBF tracking: */
124  /** Weighted sum total of all times that this router has been online.
125  */
126  unsigned long weighted_run_length;
127  /** If the router is now online (according to stability-checking rules),
128  * when did it come online? */
129  time_t start_of_run;
130  /** Sum of weights for runs in weighted_run_length. */
132  /* === For fractional uptime tracking: */
133  time_t start_of_downtime;
134  unsigned long weighted_uptime;
135  unsigned long total_weighted_time;
136 } or_history_t;
137 
138 /**
139  * This structure holds accounting needed to calculate the padding overhead.
140  */
141 typedef struct padding_counts_t {
142  /** Total number of cells we have received, including padding */
143  uint64_t read_cell_count;
144  /** Total number of cells we have sent, including padding */
146  /** Total number of CELL_PADDING cells we have received */
148  /** Total number of CELL_PADDING cells we have sent */
150  /** Total number of read cells on padding-enabled conns */
152  /** Total number of sent cells on padding-enabled conns */
154  /** Total number of read CELL_PADDING cells on padding-enabled cons */
156  /** Total number of sent CELL_PADDING cells on padding-enabled cons */
158  /** Total number of RELAY_DROP cells we have received */
160  /** Total number of RELAY_DROP cells we have sent */
162  /** The maximum number of padding timers we've seen in 24 hours */
164  /** When did we first copy padding_current into padding_published? */
165  char first_published_at[ISO_TIME_LEN+1];
167 
168 /** Holds the current values of our padding statistics.
169  * It is not published until it is transferred to padding_published. */
171 
172 /** Remains fixed for a 24 hour period, and then is replaced
173  * by a redacted copy of padding_current */
175 
176 /** When did we last multiply all routers' weighted_run_length and
177  * total_run_weights by STABILITY_ALPHA? */
178 static time_t stability_last_downrated = 0;
179 
180 /** */
181 static time_t started_tracking_stability = 0;
182 
183 /** Map from hex OR identity digest to or_history_t. */
184 static digestmap_t *history_map = NULL;
185 
186 /** Represents a state of overload stats.
187  *
188  * All the timestamps in this structure have already been rounded down to the
189  * nearest hour. */
190 typedef struct {
191  /* When did we last experience a general overload? */
192  time_t overload_general_time;
193 
194  /* When did we last experience a bandwidth-related overload? */
195  time_t overload_ratelimits_time;
196  /* How many times have we gone off the our read limits? */
197  uint64_t overload_read_count;
198  /* How many times have we gone off the our write limits? */
199  uint64_t overload_write_count;
200 
201  /* When did we last experience a file descriptor exhaustion? */
202  time_t overload_fd_exhausted_time;
203  /* How many times have we experienced a file descriptor exhaustion? */
204  uint64_t overload_fd_exhausted;
206 
207 /** Current state of overload stats */
209 
210 /** Return true if this overload happened within the last `n_hours`. */
211 static bool
212 overload_happened_recently(time_t overload_time, int n_hours)
213 {
214  /* An overload is relevant if it happened in the last 72 hours */
215  if (overload_time > approx_time() - 3600 * n_hours) {
216  return true;
217  }
218  return false;
219 }
220 
221 /* The current version of the overload stats version */
222 #define OVERLOAD_STATS_VERSION 1
223 
224 /** Returns an allocated string for server descriptor for publising information
225  * on whether we are overloaded or not. */
226 char *
228 {
229  char *result = NULL;
230  char tbuf[ISO_TIME_LEN+1];
231 
232  /* Encode the general overload */
233  if (overload_happened_recently(overload_stats.overload_general_time, 72)) {
234  format_iso_time(tbuf, overload_stats.overload_general_time);
235  tor_asprintf(&result, "overload-general %d %s\n",
236  OVERLOAD_STATS_VERSION, tbuf);
237  }
238 
239  return result;
240 }
241 
242 /** Returns an allocated string for extra-info documents for publishing
243  * overload statistics. */
244 char *
246 {
247  char *result = NULL;
248  smartlist_t *chunks = smartlist_new();
249  char tbuf[ISO_TIME_LEN+1];
250 
251  /* Add bandwidth-related overloads */
252  if (overload_happened_recently(overload_stats.overload_ratelimits_time,24)) {
253  const or_options_t *options = get_options();
254  format_iso_time(tbuf, overload_stats.overload_ratelimits_time);
255  smartlist_add_asprintf(chunks,
256  "overload-ratelimits %d %s %" PRIu64 " %" PRIu64
257  " %" PRIu64 " %" PRIu64 "\n",
258  OVERLOAD_STATS_VERSION, tbuf,
259  options->BandwidthRate, options->BandwidthBurst,
260  overload_stats.overload_read_count,
261  overload_stats.overload_write_count);
262  }
263 
264  /* Finally file descriptor overloads */
266  overload_stats.overload_fd_exhausted_time, 72)) {
267  format_iso_time(tbuf, overload_stats.overload_fd_exhausted_time);
268  smartlist_add_asprintf(chunks, "overload-fd-exhausted %d %s\n",
269  OVERLOAD_STATS_VERSION, tbuf);
270  }
271 
272  /* Bail early if we had nothing to write */
273  if (smartlist_len(chunks) == 0) {
274  goto done;
275  }
276 
277  result = smartlist_join_strings(chunks, "", 0, NULL);
278 
279  done:
280  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
281  smartlist_free(chunks);
282  return result;
283 }
284 
285 /** Round down the time in `a` to the beginning of the current hour */
286 #define SET_TO_START_OF_HOUR(a) STMT_BEGIN \
287  (a) = approx_time() - (approx_time() % 3600); \
288 STMT_END
289 
290 /** Note down an overload event of type `overload`. */
291 void
293 {
294  static time_t last_read_counted = 0;
295  static time_t last_write_counted = 0;
296 
297  switch (overload) {
298  case OVERLOAD_GENERAL:
299  SET_TO_START_OF_HOUR(overload_stats.overload_general_time);
300  break;
301  case OVERLOAD_READ: {
302  SET_TO_START_OF_HOUR(overload_stats.overload_ratelimits_time);
303  if (approx_time() >= last_read_counted + 60) { /* Count once a minute */
304  overload_stats.overload_read_count++;
305  last_read_counted = approx_time();
306  }
307  break;
308  }
309  case OVERLOAD_WRITE: {
310  SET_TO_START_OF_HOUR(overload_stats.overload_ratelimits_time);
311  if (approx_time() >= last_write_counted + 60) { /* Count once a minute */
312  overload_stats.overload_write_count++;
313  last_write_counted = approx_time();
314  }
315  break;
316  }
317  case OVERLOAD_FD_EXHAUSTED:
318  SET_TO_START_OF_HOUR(overload_stats.overload_fd_exhausted_time);
319  overload_stats.overload_fd_exhausted++;
320  break;
321  }
322 }
323 
324 /** Return the or_history_t for the OR with identity digest <b>id</b>,
325  * creating it if necessary. */
326 static or_history_t *
327 get_or_history(const char* id)
328 {
329  or_history_t *hist;
330 
331  if (tor_digest_is_zero(id))
332  return NULL;
333 
334  hist = digestmap_get(history_map, id);
335  if (!hist) {
336  hist = tor_malloc_zero(sizeof(or_history_t));
339  hist->since = hist->changed = time(NULL);
341  digestmap_set(history_map, id, hist);
342  }
343  return hist;
344 }
345 
346 /** Helper: free storage held by a single OR history entry. */
347 static void
348 free_or_history(void *_hist)
349 {
350  or_history_t *hist = _hist;
353  tor_free(hist);
354 }
355 
356 /** Initialize the static data structures for tracking history. */
357 void
359 {
360  history_map = digestmap_new();
361 }
362 
363 /** We have just decided that this router with identity digest <b>id</b> is
364  * reachable, meaning we will give it a "Running" flag for the next while. */
365 void
366 rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr,
367  const uint16_t at_port, time_t when)
368 {
369  or_history_t *hist = get_or_history(id);
370  int was_in_run = 1;
371  char tbuf[ISO_TIME_LEN+1];
372  int addr_changed, port_changed;
373 
374  tor_assert(hist);
375  tor_assert((!at_addr && !at_port) || (at_addr && at_port));
376 
377  addr_changed = at_addr && !tor_addr_is_null(&hist->last_reached_addr) &&
378  tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0;
379  port_changed = at_port && hist->last_reached_port &&
380  at_port != hist->last_reached_port;
381 
382  if (!started_tracking_stability)
383  started_tracking_stability = time(NULL);
384  if (!hist->start_of_run) {
385  hist->start_of_run = when;
386  was_in_run = 0;
387  }
388  if (hist->start_of_downtime) {
389  long down_length;
390 
391  format_local_iso_time(tbuf, hist->start_of_downtime);
392  log_info(LD_HIST, "Router %s is now Running; it had been down since %s.",
393  hex_str(id, DIGEST_LEN), tbuf);
394  if (was_in_run)
395  log_info(LD_HIST, " (Paradoxically, it was already Running too.)");
396 
397  down_length = when - hist->start_of_downtime;
398  hist->total_weighted_time += down_length;
399  hist->start_of_downtime = 0;
400  } else if (addr_changed || port_changed) {
401  /* If we're reachable, but the address changed, treat this as some
402  * downtime. */
403  int penalty = get_options()->TestingTorNetwork ? 240 : 3600;
404  networkstatus_t *ns;
405 
406  if ((ns = networkstatus_get_latest_consensus())) {
407  int fresh_interval = (int)(ns->fresh_until - ns->valid_after);
408  int live_interval = (int)(ns->valid_until - ns->valid_after);
409  /* on average, a descriptor addr change takes .5 intervals to make it
410  * into a consensus, and half a liveness period to make it to
411  * clients. */
412  penalty = (int)(fresh_interval + live_interval) / 2;
413  }
414  format_local_iso_time(tbuf, hist->start_of_run);
415  log_info(LD_HIST,"Router %s still seems Running, but its address appears "
416  "to have changed since the last time it was reachable. I'm "
417  "going to treat it as having been down for %d seconds",
418  hex_str(id, DIGEST_LEN), penalty);
419  rep_hist_note_router_unreachable(id, when-penalty);
420  rep_hist_note_router_reachable(id, NULL, 0, when);
421  } else {
422  format_local_iso_time(tbuf, hist->start_of_run);
423  if (was_in_run)
424  log_debug(LD_HIST, "Router %s is still Running; it has been Running "
425  "since %s", hex_str(id, DIGEST_LEN), tbuf);
426  else
427  log_info(LD_HIST,"Router %s is now Running; it was previously untracked",
428  hex_str(id, DIGEST_LEN));
429  }
430  if (at_addr)
431  tor_addr_copy(&hist->last_reached_addr, at_addr);
432  if (at_port)
433  hist->last_reached_port = at_port;
434 }
435 
436 /** We have just decided that this router is unreachable, meaning
437  * we are taking away its "Running" flag. */
438 void
439 rep_hist_note_router_unreachable(const char *id, time_t when)
440 {
441  or_history_t *hist = get_or_history(id);
442  char tbuf[ISO_TIME_LEN+1];
443  int was_running = 0;
444  if (!started_tracking_stability)
445  started_tracking_stability = time(NULL);
446 
447  tor_assert(hist);
448  if (hist->start_of_run) {
449  /*XXXX We could treat failed connections differently from failed
450  * connect attempts. */
451  long run_length = when - hist->start_of_run;
452  format_local_iso_time(tbuf, hist->start_of_run);
453 
454  hist->total_run_weights += 1.0;
455  hist->start_of_run = 0;
456  if (run_length < 0) {
457  unsigned long penalty = -run_length;
458 #define SUBTRACT_CLAMPED(var, penalty) \
459  do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0)
460 
461  SUBTRACT_CLAMPED(hist->weighted_run_length, penalty);
462  SUBTRACT_CLAMPED(hist->weighted_uptime, penalty);
463  } else {
464  hist->weighted_run_length += run_length;
465  hist->weighted_uptime += run_length;
466  hist->total_weighted_time += run_length;
467  }
468  was_running = 1;
469  log_info(LD_HIST, "Router %s is now non-Running: it had previously been "
470  "Running since %s. Its total weighted uptime is %lu/%lu.",
471  hex_str(id, DIGEST_LEN), tbuf, hist->weighted_uptime,
472  hist->total_weighted_time);
473  }
474  if (!hist->start_of_downtime) {
475  hist->start_of_downtime = when;
476 
477  if (!was_running)
478  log_info(LD_HIST, "Router %s is now non-Running; it was previously "
479  "untracked.", hex_str(id, DIGEST_LEN));
480  } else {
481  if (!was_running) {
482  format_local_iso_time(tbuf, hist->start_of_downtime);
483 
484  log_info(LD_HIST, "Router %s is still non-Running; it has been "
485  "non-Running since %s.", hex_str(id, DIGEST_LEN), tbuf);
486  }
487  }
488 }
489 
490 /** Mark a router with ID <b>id</b> as non-Running, and retroactively declare
491  * that it has never been running: give it no stability and no WFU. */
492 void
493 rep_hist_make_router_pessimal(const char *id, time_t when)
494 {
495  or_history_t *hist = get_or_history(id);
496  tor_assert(hist);
497 
499 
500  hist->weighted_run_length = 0;
501  hist->weighted_uptime = 0;
502 }
503 
504 /** Helper: Discount all old MTBF data, if it is time to do so. Return
505  * the time at which we should next discount MTBF data. */
506 time_t
508 {
509  digestmap_iter_t *orhist_it;
510  const char *digest1;
511  or_history_t *hist;
512  void *hist_p;
513  double alpha = 1.0;
514 
515  if (!history_map)
516  history_map = digestmap_new();
521 
522  /* Okay, we should downrate the data. By how much? */
525  alpha *= STABILITY_ALPHA;
526  }
527 
528  log_info(LD_HIST, "Discounting all old stability info by a factor of %f",
529  alpha);
530 
531  /* Multiply every w_r_l, t_r_w pair by alpha. */
532  for (orhist_it = digestmap_iter_init(history_map);
533  !digestmap_iter_done(orhist_it);
534  orhist_it = digestmap_iter_next(history_map,orhist_it)) {
535  digestmap_iter_get(orhist_it, &digest1, &hist_p);
536  hist = hist_p;
537 
538  hist->weighted_run_length =
539  (unsigned long)(hist->weighted_run_length * alpha);
540  hist->total_run_weights *= alpha;
541 
542  hist->weighted_uptime = (unsigned long)(hist->weighted_uptime * alpha);
543  hist->total_weighted_time = (unsigned long)
544  (hist->total_weighted_time * alpha);
545  }
546 
548 }
549 
550 /** Helper: Return the weighted MTBF of the router with history <b>hist</b>. */
551 static double
552 get_stability(or_history_t *hist, time_t when)
553 {
554  long total = hist->weighted_run_length;
555  double total_weights = hist->total_run_weights;
556 
557  if (hist->start_of_run) {
558  /* We're currently in a run. Let total and total_weights hold the values
559  * they would hold if the current run were to end now. */
560  total += (when-hist->start_of_run);
561  total_weights += 1.0;
562  }
563  if (total_weights < STABILITY_EPSILON) {
564  /* Round down to zero, and avoid divide-by-zero. */
565  return 0.0;
566  }
567 
568  return total / total_weights;
569 }
570 
571 /** Return the total amount of time we've been observing, with each run of
572  * time downrated by the appropriate factor. */
573 static long
575 {
576  long total = hist->total_weighted_time;
577  if (hist->start_of_run) {
578  total += (when - hist->start_of_run);
579  } else if (hist->start_of_downtime) {
580  total += (when - hist->start_of_downtime);
581  }
582  return total;
583 }
584 
585 /** Helper: Return the weighted percent-of-time-online of the router with
586  * history <b>hist</b>. */
587 static double
589 {
590  long total = hist->total_weighted_time;
591  long up = hist->weighted_uptime;
592 
593  if (hist->start_of_run) {
594  long run_length = (when - hist->start_of_run);
595  up += run_length;
596  total += run_length;
597  } else if (hist->start_of_downtime) {
598  total += (when - hist->start_of_downtime);
599  }
600 
601  if (!total) {
602  /* Avoid calling anybody's uptime infinity (which should be impossible if
603  * the code is working), or NaN (which can happen for any router we haven't
604  * observed up or down yet). */
605  return 0.0;
606  }
607 
608  return ((double) up) / total;
609 }
610 
611 /** Return how long the router whose identity digest is <b>id</b> has
612  * been reachable. Return 0 if the router is unknown or currently deemed
613  * unreachable. */
614 long
615 rep_hist_get_uptime(const char *id, time_t when)
616 {
617  or_history_t *hist = get_or_history(id);
618  if (!hist)
619  return 0;
620  if (!hist->start_of_run || when < hist->start_of_run)
621  return 0;
622  return when - hist->start_of_run;
623 }
624 
625 /** Return an estimated MTBF for the router whose identity digest is
626  * <b>id</b>. Return 0 if the router is unknown. */
627 double
628 rep_hist_get_stability(const char *id, time_t when)
629 {
630  or_history_t *hist = get_or_history(id);
631  if (!hist)
632  return 0.0;
633 
634  return get_stability(hist, when);
635 }
636 
637 /** Return an estimated percent-of-time-online for the router whose identity
638  * digest is <b>id</b>. Return 0 if the router is unknown. */
639 double
640 rep_hist_get_weighted_fractional_uptime(const char *id, time_t when)
641 {
642  or_history_t *hist = get_or_history(id);
643  if (!hist)
644  return 0.0;
645 
646  return get_weighted_fractional_uptime(hist, when);
647 }
648 
649 /** Return a number representing how long we've known about the router whose
650  * digest is <b>id</b>. Return 0 if the router is unknown.
651  *
652  * Be careful: this measure increases monotonically as we know the router for
653  * longer and longer, but it doesn't increase linearly.
654  */
655 long
656 rep_hist_get_weighted_time_known(const char *id, time_t when)
657 {
658  or_history_t *hist = get_or_history(id);
659  if (!hist)
660  return 0;
661 
662  return get_total_weighted_time(hist, when);
663 }
664 
665 /** Return true if we've been measuring MTBFs for long enough to
666  * pronounce on Stability. */
667 int
669 {
670  /* XXXX++ This doesn't do so well when we change our opinion
671  * as to whether we're tracking router stability. */
672  return started_tracking_stability < time(NULL) - 4*60*60;
673 }
674 
675 /** Log all the reliability data we have remembered, with the chosen
676  * severity.
677  */
678 void
679 rep_hist_dump_stats(time_t now, int severity)
680 {
681  digestmap_iter_t *orhist_it;
682  const char *name1, *digest1;
683  char hexdigest1[HEX_DIGEST_LEN+1];
684  or_history_t *or_history;
685  void *or_history_p;
686  const node_t *node;
687 
688  rep_history_clean(now - get_options()->RephistTrackTime);
689 
690  tor_log(severity, LD_HIST, "--------------- Dumping history information:");
691 
692  for (orhist_it = digestmap_iter_init(history_map);
693  !digestmap_iter_done(orhist_it);
694  orhist_it = digestmap_iter_next(history_map,orhist_it)) {
695  double s;
696  long stability;
697  digestmap_iter_get(orhist_it, &digest1, &or_history_p);
698  or_history = (or_history_t*) or_history_p;
699 
700  if ((node = node_get_by_id(digest1)) && node_get_nickname(node))
701  name1 = node_get_nickname(node);
702  else
703  name1 = "(unknown)";
704  base16_encode(hexdigest1, sizeof(hexdigest1), digest1, DIGEST_LEN);
705  s = get_stability(or_history, now);
706  stability = (long)s;
707  tor_log(severity, LD_HIST,
708  "OR %s [%s]: wmtbf %lu:%02lu:%02lu",
709  name1, hexdigest1,
710  stability/3600, (stability/60)%60, stability%60);
711  }
712 }
713 
714 /** Remove history info for routers/links that haven't changed since
715  * <b>before</b>.
716  */
717 void
718 rep_history_clean(time_t before)
719 {
720  int authority = authdir_mode(get_options());
721  or_history_t *or_history;
722  void *or_history_p;
723  digestmap_iter_t *orhist_it;
724  const char *d1;
725 
726  orhist_it = digestmap_iter_init(history_map);
727  while (!digestmap_iter_done(orhist_it)) {
728  int should_remove;
729  digestmap_iter_get(orhist_it, &d1, &or_history_p);
730  or_history = or_history_p;
731 
732  should_remove = authority ?
733  (or_history->total_run_weights < STABILITY_EPSILON &&
734  !or_history->start_of_run)
735  : (or_history->changed < before);
736  if (should_remove) {
737  orhist_it = digestmap_iter_next_rmv(history_map, orhist_it);
738  free_or_history(or_history);
739  continue;
740  }
741  orhist_it = digestmap_iter_next(history_map, orhist_it);
742  }
743 }
744 
745 /** Write MTBF data to disk. Return 0 on success, negative on failure.
746  *
747  * If <b>missing_means_down</b>, then if we're about to write an entry
748  * that is still considered up but isn't in our routerlist, consider it
749  * to be down. */
750 int
751 rep_hist_record_mtbf_data(time_t now, int missing_means_down)
752 {
753  char time_buf[ISO_TIME_LEN+1];
754 
755  digestmap_iter_t *orhist_it;
756  const char *digest;
757  void *or_history_p;
758  or_history_t *hist;
759  open_file_t *open_file = NULL;
760  FILE *f;
761 
762  {
763  char *filename = get_datadir_fname("router-stability");
764  f = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE|O_TEXT, 0600,
765  &open_file);
766  tor_free(filename);
767  if (!f)
768  return -1;
769  }
770 
771  /* File format is:
772  * FormatLine *KeywordLine Data
773  *
774  * FormatLine = "format 1" NL
775  * KeywordLine = Keyword SP Arguments NL
776  * Data = "data" NL *RouterMTBFLine "." NL
777  * RouterMTBFLine = Fingerprint SP WeightedRunLen SP
778  * TotalRunWeights [SP S=StartRunTime] NL
779  */
780 #define PUT(s) STMT_BEGIN if (fputs((s),f)<0) goto err; STMT_END
781 #define PRINTF(args) STMT_BEGIN if (fprintf args <0) goto err; STMT_END
782 
783  PUT("format 2\n");
784 
785  format_iso_time(time_buf, time(NULL));
786  PRINTF((f, "stored-at %s\n", time_buf));
787 
788  if (started_tracking_stability) {
789  format_iso_time(time_buf, started_tracking_stability);
790  PRINTF((f, "tracked-since %s\n", time_buf));
791  }
794  PRINTF((f, "last-downrated %s\n", time_buf));
795  }
796 
797  PUT("data\n");
798 
799  /* XXX Nick: now bridge auths record this for all routers too.
800  * Should we make them record it only for bridge routers? -RD
801  * Not for 0.2.0. -NM */
802  for (orhist_it = digestmap_iter_init(history_map);
803  !digestmap_iter_done(orhist_it);
804  orhist_it = digestmap_iter_next(history_map,orhist_it)) {
805  char dbuf[HEX_DIGEST_LEN+1];
806  const char *t = NULL;
807  digestmap_iter_get(orhist_it, &digest, &or_history_p);
808  hist = (or_history_t*) or_history_p;
809 
810  base16_encode(dbuf, sizeof(dbuf), digest, DIGEST_LEN);
811 
812  if (missing_means_down && hist->start_of_run &&
814  /* We think this relay is running, but it's not listed in our
815  * consensus. Somehow it fell out without telling us it went
816  * down. Complain and also correct it. */
817  log_info(LD_HIST,
818  "Relay '%s' is listed as up in rephist, but it's not in "
819  "our routerlist. Correcting.", dbuf);
821  }
822 
823  PRINTF((f, "R %s\n", dbuf));
824  if (hist->start_of_run > 0) {
825  format_iso_time(time_buf, hist->start_of_run);
826  t = time_buf;
827  }
828  PRINTF((f, "+MTBF %lu %.5f%s%s\n",
830  t ? " S=" : "", t ? t : ""));
831  t = NULL;
832  if (hist->start_of_downtime > 0) {
833  format_iso_time(time_buf, hist->start_of_downtime);
834  t = time_buf;
835  }
836  PRINTF((f, "+WFU %lu %lu%s%s\n",
837  hist->weighted_uptime, hist->total_weighted_time,
838  t ? " S=" : "", t ? t : ""));
839  }
840 
841  PUT(".\n");
842 
843 #undef PUT
844 #undef PRINTF
845 
846  return finish_writing_to_file(open_file);
847  err:
848  abort_writing_to_file(open_file);
849  return -1;
850 }
851 
852 /** Helper: return the first j >= i such that !strcmpstart(sl[j], prefix) and
853  * such that no line sl[k] with i <= k < j starts with "R ". Return -1 if no
854  * such line exists. */
855 static int
856 find_next_with(smartlist_t *sl, int i, const char *prefix)
857 {
858  for ( ; i < smartlist_len(sl); ++i) {
859  const char *line = smartlist_get(sl, i);
860  if (!strcmpstart(line, prefix))
861  return i;
862  if (!strcmpstart(line, "R "))
863  return -1;
864  }
865  return -1;
866 }
867 
868 /** How many bad times has parse_possibly_bad_iso_time() parsed? */
869 static int n_bogus_times = 0;
870 /** Parse the ISO-formatted time in <b>s</b> into *<b>time_out</b>, but
871  * round any pre-1970 date to Jan 1, 1970. */
872 static int
873 parse_possibly_bad_iso_time(const char *s, time_t *time_out)
874 {
875  int year;
876  char b[5];
877  strlcpy(b, s, sizeof(b));
878  b[4] = '\0';
879  year = (int)tor_parse_long(b, 10, 0, INT_MAX, NULL, NULL);
880  if (year < 1970) {
881  *time_out = 0;
882  ++n_bogus_times;
883  return 0;
884  } else
885  return parse_iso_time(s, time_out);
886 }
887 
888 /** We've read a time <b>t</b> from a file stored at <b>stored_at</b>, which
889  * says we started measuring at <b>started_measuring</b>. Return a new number
890  * that's about as much before <b>now</b> as <b>t</b> was before
891  * <b>stored_at</b>.
892  */
893 static inline time_t
894 correct_time(time_t t, time_t now, time_t stored_at, time_t started_measuring)
895 {
896  if (t < started_measuring - 24*60*60*365)
897  return 0;
898  else if (t < started_measuring)
899  return started_measuring;
900  else if (t > stored_at)
901  return 0;
902  else {
903  long run_length = stored_at - t;
904  t = (time_t)(now - run_length);
905  if (t < started_measuring)
906  t = started_measuring;
907  return t;
908  }
909 }
910 
911 /** Load MTBF data from disk. Returns 0 on success or recoverable error, -1
912  * on failure. */
913 int
915 {
916  /* XXXX won't handle being called while history is already populated. */
917  smartlist_t *lines;
918  const char *line = NULL;
919  int r=0, i;
920  time_t last_downrated = 0, stored_at = 0, tracked_since = 0;
921  time_t latest_possible_start = now;
922  long format = -1;
923 
924  {
925  char *filename = get_datadir_fname("router-stability");
926  char *d = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
927  tor_free(filename);
928  if (!d)
929  return -1;
930  lines = smartlist_new();
931  smartlist_split_string(lines, d, "\n", SPLIT_SKIP_SPACE, 0);
932  tor_free(d);
933  }
934 
935  {
936  const char *firstline;
937  if (smartlist_len(lines)>4) {
938  firstline = smartlist_get(lines, 0);
939  if (!strcmpstart(firstline, "format "))
940  format = tor_parse_long(firstline+strlen("format "),
941  10, -1, LONG_MAX, NULL, NULL);
942  }
943  }
944  if (format != 1 && format != 2) {
945  log_warn(LD_HIST,
946  "Unrecognized format in mtbf history file. Skipping.");
947  goto err;
948  }
949  for (i = 1; i < smartlist_len(lines); ++i) {
950  line = smartlist_get(lines, i);
951  if (!strcmp(line, "data"))
952  break;
953  if (!strcmpstart(line, "last-downrated ")) {
954  if (parse_iso_time(line+strlen("last-downrated "), &last_downrated)<0)
955  log_warn(LD_HIST,"Couldn't parse downrate time in mtbf "
956  "history file.");
957  }
958  if (!strcmpstart(line, "stored-at ")) {
959  if (parse_iso_time(line+strlen("stored-at "), &stored_at)<0)
960  log_warn(LD_HIST,"Couldn't parse stored time in mtbf "
961  "history file.");
962  }
963  if (!strcmpstart(line, "tracked-since ")) {
964  if (parse_iso_time(line+strlen("tracked-since "), &tracked_since)<0)
965  log_warn(LD_HIST,"Couldn't parse started-tracking time in mtbf "
966  "history file.");
967  }
968  }
969  if (last_downrated > now)
970  last_downrated = now;
971  if (tracked_since > now)
972  tracked_since = now;
973 
974  if (!stored_at) {
975  log_warn(LD_HIST, "No stored time recorded.");
976  goto err;
977  }
978 
979  if (line && !strcmp(line, "data"))
980  ++i;
981 
982  n_bogus_times = 0;
983 
984  for (; i < smartlist_len(lines); ++i) {
985  char digest[DIGEST_LEN];
986  char hexbuf[HEX_DIGEST_LEN+1];
987  char mtbf_timebuf[ISO_TIME_LEN+1];
988  char wfu_timebuf[ISO_TIME_LEN+1];
989  time_t start_of_run = 0;
990  time_t start_of_downtime = 0;
991  int have_mtbf = 0, have_wfu = 0;
992  long wrl = 0;
993  double trw = 0;
994  long wt_uptime = 0, total_wt_time = 0;
995  int n;
996  or_history_t *hist;
997  line = smartlist_get(lines, i);
998  if (!strcmp(line, "."))
999  break;
1000 
1001  mtbf_timebuf[0] = '\0';
1002  wfu_timebuf[0] = '\0';
1003 
1004  if (format == 1) {
1005  n = tor_sscanf(line, "%40s %ld %lf S=%10s %8s",
1006  hexbuf, &wrl, &trw, mtbf_timebuf, mtbf_timebuf+11);
1007  if (n != 3 && n != 5) {
1008  log_warn(LD_HIST, "Couldn't scan line %s", escaped(line));
1009  continue;
1010  }
1011  have_mtbf = 1;
1012  } else {
1013  // format == 2.
1014  int mtbf_idx, wfu_idx;
1015  if (strcmpstart(line, "R ") || strlen(line) < 2+HEX_DIGEST_LEN)
1016  continue;
1017  strlcpy(hexbuf, line+2, sizeof(hexbuf));
1018  mtbf_idx = find_next_with(lines, i+1, "+MTBF ");
1019  wfu_idx = find_next_with(lines, i+1, "+WFU ");
1020  if (mtbf_idx >= 0) {
1021  const char *mtbfline = smartlist_get(lines, mtbf_idx);
1022  n = tor_sscanf(mtbfline, "+MTBF %lu %lf S=%10s %8s",
1023  &wrl, &trw, mtbf_timebuf, mtbf_timebuf+11);
1024  if (n == 2 || n == 4) {
1025  have_mtbf = 1;
1026  } else {
1027  log_warn(LD_HIST, "Couldn't scan +MTBF line %s",
1028  escaped(mtbfline));
1029  }
1030  }
1031  if (wfu_idx >= 0) {
1032  const char *wfuline = smartlist_get(lines, wfu_idx);
1033  n = tor_sscanf(wfuline, "+WFU %lu %lu S=%10s %8s",
1034  &wt_uptime, &total_wt_time,
1035  wfu_timebuf, wfu_timebuf+11);
1036  if (n == 2 || n == 4) {
1037  have_wfu = 1;
1038  } else {
1039  log_warn(LD_HIST, "Couldn't scan +WFU line %s", escaped(wfuline));
1040  }
1041  }
1042  if (wfu_idx > i)
1043  i = wfu_idx;
1044  if (mtbf_idx > i)
1045  i = mtbf_idx;
1046  }
1047  if (base16_decode(digest, DIGEST_LEN,
1048  hexbuf, HEX_DIGEST_LEN) != DIGEST_LEN) {
1049  log_warn(LD_HIST, "Couldn't hex string %s", escaped(hexbuf));
1050  continue;
1051  }
1052  hist = get_or_history(digest);
1053  if (!hist)
1054  continue;
1055 
1056  if (have_mtbf) {
1057  if (mtbf_timebuf[0]) {
1058  mtbf_timebuf[10] = ' ';
1059  if (parse_possibly_bad_iso_time(mtbf_timebuf, &start_of_run)<0)
1060  log_warn(LD_HIST, "Couldn't parse time %s",
1061  escaped(mtbf_timebuf));
1062  }
1063  hist->start_of_run = correct_time(start_of_run, now, stored_at,
1064  tracked_since);
1065  if (hist->start_of_run < latest_possible_start + wrl)
1066  latest_possible_start = (time_t)(hist->start_of_run - wrl);
1067 
1068  hist->weighted_run_length = wrl;
1069  hist->total_run_weights = trw;
1070  }
1071  if (have_wfu) {
1072  if (wfu_timebuf[0]) {
1073  wfu_timebuf[10] = ' ';
1074  if (parse_possibly_bad_iso_time(wfu_timebuf, &start_of_downtime)<0)
1075  log_warn(LD_HIST, "Couldn't parse time %s", escaped(wfu_timebuf));
1076  }
1077  }
1078  hist->start_of_downtime = correct_time(start_of_downtime, now, stored_at,
1079  tracked_since);
1080  hist->weighted_uptime = wt_uptime;
1081  hist->total_weighted_time = total_wt_time;
1082  }
1083  if (strcmp(line, "."))
1084  log_warn(LD_HIST, "Truncated MTBF file.");
1085 
1086  if (tracked_since < 86400*365) /* Recover from insanely early value. */
1087  tracked_since = latest_possible_start;
1088 
1089  stability_last_downrated = last_downrated;
1090  started_tracking_stability = tracked_since;
1091 
1092  goto done;
1093  err:
1094  r = -1;
1095  done:
1096  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
1097  smartlist_free(lines);
1098  return r;
1099 }
1100 
1101 /*** Exit port statistics ***/
1102 
1103 /* Some constants */
1104 /** To what multiple should byte numbers be rounded up? */
1105 #define EXIT_STATS_ROUND_UP_BYTES 1024
1106 /** To what multiple should stream counts be rounded up? */
1107 #define EXIT_STATS_ROUND_UP_STREAMS 4
1108 /** Number of TCP ports */
1109 #define EXIT_STATS_NUM_PORTS 65536
1110 /** Top n ports that will be included in exit stats. */
1111 #define EXIT_STATS_TOP_N_PORTS 10
1112 
1113 /* The following data structures are arrays and no fancy smartlists or maps,
1114  * so that all write operations can be done in constant time. This comes at
1115  * the price of some memory (1.25 MB) and linear complexity when writing
1116  * stats for measuring relays. */
1117 /** Number of bytes read in current period by exit port */
1118 static uint64_t *exit_bytes_read = NULL;
1119 /** Number of bytes written in current period by exit port */
1120 static uint64_t *exit_bytes_written = NULL;
1121 /** Number of streams opened in current period by exit port */
1122 static uint32_t *exit_streams = NULL;
1123 
1124 /** Start time of exit stats or 0 if we're not collecting exit stats. */
1126 
1127 /** Initialize exit port stats. */
1128 void
1130 {
1132  exit_bytes_read = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t));
1133  exit_bytes_written = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t));
1134  exit_streams = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint32_t));
1135 }
1136 
1137 /** Reset counters for exit port statistics. */
1138 void
1140 {
1142  memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t));
1143  memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t));
1144  memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t));
1145 }
1146 
1147 /** Stop collecting exit port stats in a way that we can re-start doing
1148  * so in rep_hist_exit_stats_init(). */
1149 void
1151 {
1156 }
1157 
1158 /** Helper for qsort: compare two ints. Does not handle overflow properly,
1159  * but works fine for sorting an array of port numbers, which is what we use
1160  * it for. */
1161 static int
1162 compare_int_(const void *x, const void *y)
1163 {
1164  return (*(int*)x - *(int*)y);
1165 }
1166 
1167 /** Return a newly allocated string containing the exit port statistics
1168  * until <b>now</b>, or NULL if we're not collecting exit stats. Caller
1169  * must ensure start_of_exit_stats_interval is in the past. */
1170 char *
1172 {
1173  int i, j, top_elements = 0, cur_min_idx = 0, cur_port;
1174  uint64_t top_bytes[EXIT_STATS_TOP_N_PORTS];
1175  int top_ports[EXIT_STATS_TOP_N_PORTS];
1176  uint64_t cur_bytes = 0, other_read = 0, other_written = 0,
1177  total_read = 0, total_written = 0;
1178  uint32_t total_streams = 0, other_streams = 0;
1179  smartlist_t *written_strings, *read_strings, *streams_strings;
1180  char *written_string, *read_string, *streams_string;
1181  char t[ISO_TIME_LEN+1];
1182  char *result;
1183 
1185  return NULL; /* Not initialized. */
1186 
1188 
1189  /* Go through all ports to find the n ports that saw most written and
1190  * read bytes.
1191  *
1192  * Invariant: at the end of the loop for iteration i,
1193  * total_read is the sum of all exit_bytes_read[0..i]
1194  * total_written is the sum of all exit_bytes_written[0..i]
1195  * total_stream is the sum of all exit_streams[0..i]
1196  *
1197  * top_elements = MAX(EXIT_STATS_TOP_N_PORTS,
1198  * #{j | 0 <= j <= i && volume(i) > 0})
1199  *
1200  * For all 0 <= j < top_elements,
1201  * top_bytes[j] > 0
1202  * 0 <= top_ports[j] <= 65535
1203  * top_bytes[j] = volume(top_ports[j])
1204  *
1205  * There is no j in 0..i and k in 0..top_elements such that:
1206  * volume(j) > top_bytes[k] AND j is not in top_ports[0..top_elements]
1207  *
1208  * There is no j!=cur_min_idx in 0..top_elements such that:
1209  * top_bytes[j] < top_bytes[cur_min_idx]
1210  *
1211  * where volume(x) == exit_bytes_read[x]+exit_bytes_written[x]
1212  *
1213  * Worst case: O(EXIT_STATS_NUM_PORTS * EXIT_STATS_TOP_N_PORTS)
1214  */
1215  for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) {
1216  total_read += exit_bytes_read[i];
1217  total_written += exit_bytes_written[i];
1218  total_streams += exit_streams[i];
1219  cur_bytes = exit_bytes_read[i] + exit_bytes_written[i];
1220  if (cur_bytes == 0) {
1221  continue;
1222  }
1223  if (top_elements < EXIT_STATS_TOP_N_PORTS) {
1224  top_bytes[top_elements] = cur_bytes;
1225  top_ports[top_elements++] = i;
1226  } else if (cur_bytes > top_bytes[cur_min_idx]) {
1227  top_bytes[cur_min_idx] = cur_bytes;
1228  top_ports[cur_min_idx] = i;
1229  } else {
1230  continue;
1231  }
1232  cur_min_idx = 0;
1233  for (j = 1; j < top_elements; j++) {
1234  if (top_bytes[j] < top_bytes[cur_min_idx]) {
1235  cur_min_idx = j;
1236  }
1237  }
1238  }
1239 
1240  /* Add observations of top ports to smartlists. */
1241  written_strings = smartlist_new();
1242  read_strings = smartlist_new();
1243  streams_strings = smartlist_new();
1244  other_read = total_read;
1245  other_written = total_written;
1246  other_streams = total_streams;
1247  /* Sort the ports; this puts them out of sync with top_bytes, but we
1248  * won't be using top_bytes again anyway */
1249  qsort(top_ports, top_elements, sizeof(int), compare_int_);
1250  for (j = 0; j < top_elements; j++) {
1251  cur_port = top_ports[j];
1252  if (exit_bytes_written[cur_port] > 0) {
1253  uint64_t num = round_uint64_to_next_multiple_of(
1254  exit_bytes_written[cur_port],
1256  num /= 1024;
1257  smartlist_add_asprintf(written_strings, "%d=%"PRIu64,
1258  cur_port, (num));
1259  other_written -= exit_bytes_written[cur_port];
1260  }
1261  if (exit_bytes_read[cur_port] > 0) {
1262  uint64_t num = round_uint64_to_next_multiple_of(
1263  exit_bytes_read[cur_port],
1265  num /= 1024;
1266  smartlist_add_asprintf(read_strings, "%d=%"PRIu64,
1267  cur_port, (num));
1268  other_read -= exit_bytes_read[cur_port];
1269  }
1270  if (exit_streams[cur_port] > 0) {
1271  uint32_t num = round_uint32_to_next_multiple_of(
1272  exit_streams[cur_port],
1274  smartlist_add_asprintf(streams_strings, "%d=%u", cur_port, num);
1275  other_streams -= exit_streams[cur_port];
1276  }
1277  }
1278 
1279  /* Add observations of other ports in a single element. */
1280  other_written = round_uint64_to_next_multiple_of(other_written,
1282  other_written /= 1024;
1283  smartlist_add_asprintf(written_strings, "other=%"PRIu64,
1284  (other_written));
1285  other_read = round_uint64_to_next_multiple_of(other_read,
1287  other_read /= 1024;
1288  smartlist_add_asprintf(read_strings, "other=%"PRIu64,
1289  (other_read));
1290  other_streams = round_uint32_to_next_multiple_of(other_streams,
1292  smartlist_add_asprintf(streams_strings, "other=%u", other_streams);
1293 
1294  /* Join all observations in single strings. */
1295  written_string = smartlist_join_strings(written_strings, ",", 0, NULL);
1296  read_string = smartlist_join_strings(read_strings, ",", 0, NULL);
1297  streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL);
1298  SMARTLIST_FOREACH(written_strings, char *, cp, tor_free(cp));
1299  SMARTLIST_FOREACH(read_strings, char *, cp, tor_free(cp));
1300  SMARTLIST_FOREACH(streams_strings, char *, cp, tor_free(cp));
1301  smartlist_free(written_strings);
1302  smartlist_free(read_strings);
1303  smartlist_free(streams_strings);
1304 
1305  /* Put everything together. */
1306  format_iso_time(t, now);
1307  tor_asprintf(&result, "exit-stats-end %s (%d s)\n"
1308  "exit-kibibytes-written %s\n"
1309  "exit-kibibytes-read %s\n"
1310  "exit-streams-opened %s\n",
1311  t, (unsigned) (now - start_of_exit_stats_interval),
1312  written_string,
1313  read_string,
1314  streams_string);
1315  tor_free(written_string);
1316  tor_free(read_string);
1317  tor_free(streams_string);
1318  return result;
1319 }
1320 
1321 /** If 24 hours have passed since the beginning of the current exit port
1322  * stats period, write exit stats to $DATADIR/stats/exit-stats (possibly
1323  * overwriting an existing file) and reset counters. Return when we would
1324  * next want to write exit stats or 0 if we never want to write. */
1325 time_t
1327 {
1328  char *str = NULL;
1329 
1331  return 0; /* Not initialized. */
1332  if (start_of_exit_stats_interval + WRITE_STATS_INTERVAL > now)
1333  goto done; /* Not ready to write. */
1334 
1335  log_info(LD_HIST, "Writing exit port statistics to disk.");
1336 
1337  /* Generate history string. */
1338  str = rep_hist_format_exit_stats(now);
1339 
1340  /* Reset counters. */
1342 
1343  /* Try to write to disk. */
1344  if (!check_or_create_data_subdir("stats")) {
1345  write_to_data_subdir("stats", "exit-stats", str, "exit port statistics");
1346  }
1347 
1348  done:
1349  tor_free(str);
1350  return start_of_exit_stats_interval + WRITE_STATS_INTERVAL;
1351 }
1352 
1353 /** Note that we wrote <b>num_written</b> bytes and read <b>num_read</b>
1354  * bytes to/from an exit connection to <b>port</b>. */
1355 void
1356 rep_hist_note_exit_bytes(uint16_t port, size_t num_written,
1357  size_t num_read)
1358 {
1360  return; /* Not initialized. */
1361  exit_bytes_written[port] += num_written;
1362  exit_bytes_read[port] += num_read;
1363  log_debug(LD_HIST, "Written %lu bytes and read %lu bytes to/from an "
1364  "exit connection to port %d.",
1365  (unsigned long)num_written, (unsigned long)num_read, port);
1366 }
1367 
1368 /** Note that we opened an exit stream to <b>port</b>. */
1369 void
1371 {
1373  return; /* Not initialized. */
1374  exit_streams[port]++;
1375  log_debug(LD_HIST, "Opened exit stream to port %d", port);
1376 }
1377 
1378 /*** cell statistics ***/
1379 
1380 /** Start of the current buffer stats interval or 0 if we're not
1381  * collecting buffer statistics. */
1383 
1384 /** Initialize buffer stats. */
1385 void
1387 {
1389 }
1390 
1391 /** Statistics from a single circuit. Collected when the circuit closes, or
1392  * when we flush statistics to disk. */
1393 typedef struct circ_buffer_stats_t {
1394  /** Average number of cells in the circuit's queue */
1396  /** Average time a cell waits in the queue. */
1398  /** Total number of cells sent over this circuit */
1401 
1402 /** List of circ_buffer_stats_t. */
1404 
1405 /** Remember cell statistics <b>mean_num_cells_in_queue</b>,
1406  * <b>mean_time_cells_in_queue</b>, and <b>processed_cells</b> of a
1407  * circuit. */
1408 void
1409 rep_hist_add_buffer_stats(double mean_num_cells_in_queue,
1410  double mean_time_cells_in_queue, uint32_t processed_cells)
1411 {
1412  circ_buffer_stats_t *stats;
1414  return; /* Not initialized. */
1415  stats = tor_malloc_zero(sizeof(circ_buffer_stats_t));
1416  stats->mean_num_cells_in_queue = mean_num_cells_in_queue;
1417  stats->mean_time_cells_in_queue = mean_time_cells_in_queue;
1418  stats->processed_cells = processed_cells;
1422 }
1423 
1424 /** Remember cell statistics for circuit <b>circ</b> at time
1425  * <b>end_of_interval</b> and reset cell counters in case the circuit
1426  * remains open in the next measurement interval. */
1427 void
1428 rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval)
1429 {
1430  time_t start_of_interval;
1431  int interval_length;
1432  or_circuit_t *orcirc;
1433  double mean_num_cells_in_queue, mean_time_cells_in_queue;
1434  uint32_t processed_cells;
1435  if (CIRCUIT_IS_ORIGIN(circ))
1436  return;
1437  orcirc = TO_OR_CIRCUIT(circ);
1438  if (!orcirc->processed_cells)
1439  return;
1440  start_of_interval = (circ->timestamp_created.tv_sec >
1442  (time_t)circ->timestamp_created.tv_sec :
1444  interval_length = (int) (end_of_interval - start_of_interval);
1445  if (interval_length <= 0)
1446  return;
1447  processed_cells = orcirc->processed_cells;
1448  /* 1000.0 for s -> ms; 2.0 because of app-ward and exit-ward queues */
1449  mean_num_cells_in_queue = (double) orcirc->total_cell_waiting_time /
1450  (double) interval_length / 1000.0 / 2.0;
1451  mean_time_cells_in_queue =
1452  (double) orcirc->total_cell_waiting_time /
1453  (double) orcirc->processed_cells;
1454  orcirc->total_cell_waiting_time = 0;
1455  orcirc->processed_cells = 0;
1456  rep_hist_add_buffer_stats(mean_num_cells_in_queue,
1457  mean_time_cells_in_queue,
1458  processed_cells);
1459 }
1460 
1461 /** Sorting helper: return -1, 1, or 0 based on comparison of two
1462  * circ_buffer_stats_t */
1463 static int
1464 buffer_stats_compare_entries_(const void **_a, const void **_b)
1465 {
1466  const circ_buffer_stats_t *a = *_a, *b = *_b;
1467  if (a->processed_cells < b->processed_cells)
1468  return 1;
1469  else if (a->processed_cells > b->processed_cells)
1470  return -1;
1471  else
1472  return 0;
1473 }
1474 
1475 /** Stop collecting cell stats in a way that we can re-start doing so in
1476  * rep_hist_buffer_stats_init(). */
1477 void
1479 {
1481 }
1482 
1483 /** Clear history of circuit statistics and set the measurement interval
1484  * start to <b>now</b>. */
1485 void
1487 {
1491  stats, tor_free(stats));
1494 }
1495 
1496 /** Return a newly allocated string containing the buffer statistics until
1497  * <b>now</b>, or NULL if we're not collecting buffer stats. Caller must
1498  * ensure start_of_buffer_stats_interval is in the past. */
1499 char *
1501 {
1502 #define SHARES 10
1503  uint64_t processed_cells[SHARES];
1504  uint32_t circs_in_share[SHARES];
1505  int number_of_circuits, i;
1506  double queued_cells[SHARES], time_in_queue[SHARES];
1507  smartlist_t *processed_cells_strings, *queued_cells_strings,
1508  *time_in_queue_strings;
1509  char *processed_cells_string, *queued_cells_string,
1510  *time_in_queue_string;
1511  char t[ISO_TIME_LEN+1];
1512  char *result;
1513 
1515  return NULL; /* Not initialized. */
1516 
1518 
1519  /* Calculate deciles if we saw at least one circuit. */
1520  memset(processed_cells, 0, SHARES * sizeof(uint64_t));
1521  memset(circs_in_share, 0, SHARES * sizeof(uint32_t));
1522  memset(queued_cells, 0, SHARES * sizeof(double));
1523  memset(time_in_queue, 0, SHARES * sizeof(double));
1526  number_of_circuits = smartlist_len(circuits_for_buffer_stats);
1527  if (number_of_circuits > 0) {
1530  i = 0;
1532  circ_buffer_stats_t *, stats)
1533  {
1534  int share = i++ * SHARES / number_of_circuits;
1535  processed_cells[share] += stats->processed_cells;
1536  queued_cells[share] += stats->mean_num_cells_in_queue;
1537  time_in_queue[share] += stats->mean_time_cells_in_queue;
1538  circs_in_share[share]++;
1539  }
1540  SMARTLIST_FOREACH_END(stats);
1541  }
1542 
1543  /* Write deciles to strings. */
1544  processed_cells_strings = smartlist_new();
1545  queued_cells_strings = smartlist_new();
1546  time_in_queue_strings = smartlist_new();
1547  for (i = 0; i < SHARES; i++) {
1548  smartlist_add_asprintf(processed_cells_strings,
1549  "%"PRIu64, !circs_in_share[i] ? 0 :
1550  (processed_cells[i] /
1551  circs_in_share[i]));
1552  }
1553  for (i = 0; i < SHARES; i++) {
1554  smartlist_add_asprintf(queued_cells_strings, "%.2f",
1555  circs_in_share[i] == 0 ? 0.0 :
1556  queued_cells[i] / (double) circs_in_share[i]);
1557  }
1558  for (i = 0; i < SHARES; i++) {
1559  smartlist_add_asprintf(time_in_queue_strings, "%.0f",
1560  circs_in_share[i] == 0 ? 0.0 :
1561  time_in_queue[i] / (double) circs_in_share[i]);
1562  }
1563 
1564  /* Join all observations in single strings. */
1565  processed_cells_string = smartlist_join_strings(processed_cells_strings,
1566  ",", 0, NULL);
1567  queued_cells_string = smartlist_join_strings(queued_cells_strings,
1568  ",", 0, NULL);
1569  time_in_queue_string = smartlist_join_strings(time_in_queue_strings,
1570  ",", 0, NULL);
1571  SMARTLIST_FOREACH(processed_cells_strings, char *, cp, tor_free(cp));
1572  SMARTLIST_FOREACH(queued_cells_strings, char *, cp, tor_free(cp));
1573  SMARTLIST_FOREACH(time_in_queue_strings, char *, cp, tor_free(cp));
1574  smartlist_free(processed_cells_strings);
1575  smartlist_free(queued_cells_strings);
1576  smartlist_free(time_in_queue_strings);
1577 
1578  /* Put everything together. */
1579  format_iso_time(t, now);
1580  tor_asprintf(&result, "cell-stats-end %s (%d s)\n"
1581  "cell-processed-cells %s\n"
1582  "cell-queued-cells %s\n"
1583  "cell-time-in-queue %s\n"
1584  "cell-circuits-per-decile %d\n",
1585  t, (unsigned) (now - start_of_buffer_stats_interval),
1586  processed_cells_string,
1587  queued_cells_string,
1588  time_in_queue_string,
1589  CEIL_DIV(number_of_circuits, SHARES));
1590  tor_free(processed_cells_string);
1591  tor_free(queued_cells_string);
1592  tor_free(time_in_queue_string);
1593  return result;
1594 #undef SHARES
1595 }
1596 
1597 /** If 24 hours have passed since the beginning of the current buffer
1598  * stats period, write buffer stats to $DATADIR/stats/buffer-stats
1599  * (possibly overwriting an existing file) and reset counters. Return
1600  * when we would next want to write buffer stats or 0 if we never want to
1601  * write. */
1602 time_t
1604 {
1605  char *str = NULL;
1606 
1608  return 0; /* Not initialized. */
1609  if (start_of_buffer_stats_interval + WRITE_STATS_INTERVAL > now)
1610  goto done; /* Not ready to write */
1611 
1612  /* Add open circuits to the history. */
1614  rep_hist_buffer_stats_add_circ(circ, now);
1615  }
1616  SMARTLIST_FOREACH_END(circ);
1617 
1618  /* Generate history string. */
1619  str = rep_hist_format_buffer_stats(now);
1620 
1621  /* Reset both buffer history and counters of open circuits. */
1623 
1624  /* Try to write to disk. */
1625  if (!check_or_create_data_subdir("stats")) {
1626  write_to_data_subdir("stats", "buffer-stats", str, "buffer statistics");
1627  }
1628 
1629  done:
1630  tor_free(str);
1631  return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL;
1632 }
1633 
1634 /*** Descriptor serving statistics ***/
1635 
1636 /** Digestmap to track which descriptors were downloaded this stats
1637  * collection interval. It maps descriptor digest to pointers to 1,
1638  * effectively turning this into a list. */
1639 static digestmap_t *served_descs = NULL;
1640 
1641 /** Number of how many descriptors were downloaded in total during this
1642  * interval. */
1643 static unsigned long total_descriptor_downloads;
1644 
1645 /** Start time of served descs stats or 0 if we're not collecting those. */
1647 
1648 /** Initialize descriptor stats. */
1649 void
1651 {
1652  if (served_descs) {
1653  log_warn(LD_BUG, "Called rep_hist_desc_stats_init() when desc stats were "
1654  "already initialized. This is probably harmless.");
1655  return; // Already initialized
1656  }
1657  served_descs = digestmap_new();
1660 }
1661 
1662 /** Reset served descs stats to empty, starting a new interval <b>now</b>. */
1663 static void
1665 {
1668 }
1669 
1670 /** Stop collecting served descs stats, so that rep_hist_desc_stats_init() is
1671  * safe to be called again. */
1672 void
1674 {
1675  digestmap_free(served_descs, NULL);
1676  served_descs = NULL;
1679 }
1680 
1681 /** Helper for rep_hist_desc_stats_write(). Return a newly allocated string
1682  * containing the served desc statistics until now, or NULL if we're not
1683  * collecting served desc stats. Caller must ensure that now is not before
1684  * start_of_served_descs_stats_interval. */
1685 static char *
1687 {
1688  char t[ISO_TIME_LEN+1];
1689  char *result;
1690 
1691  digestmap_iter_t *iter;
1692  const char *key;
1693  void *val;
1694  unsigned size;
1695  int *vals, max = 0, q3 = 0, md = 0, q1 = 0, min = 0;
1696  int n = 0;
1697 
1699  return NULL;
1700 
1701  size = digestmap_size(served_descs);
1702  if (size > 0) {
1703  vals = tor_calloc(size, sizeof(int));
1704  for (iter = digestmap_iter_init(served_descs);
1705  !digestmap_iter_done(iter);
1706  iter = digestmap_iter_next(served_descs, iter)) {
1707  uintptr_t count;
1708  digestmap_iter_get(iter, &key, &val);
1709  count = (uintptr_t)val;
1710  vals[n++] = (int)count;
1711  (void)key;
1712  }
1713  max = find_nth_int(vals, size, size-1);
1714  q3 = find_nth_int(vals, size, (3*size-1)/4);
1715  md = find_nth_int(vals, size, (size-1)/2);
1716  q1 = find_nth_int(vals, size, (size-1)/4);
1717  min = find_nth_int(vals, size, 0);
1718  tor_free(vals);
1719  }
1720 
1721  format_iso_time(t, now);
1722 
1723  tor_asprintf(&result,
1724  "served-descs-stats-end %s (%d s) total=%lu unique=%u "
1725  "max=%d q3=%d md=%d q1=%d min=%d\n",
1726  t,
1727  (unsigned) (now - start_of_served_descs_stats_interval),
1729  size, max, q3, md, q1, min);
1730 
1731  return result;
1732 }
1733 
1734 /** If WRITE_STATS_INTERVAL seconds have passed since the beginning of
1735  * the current served desc stats interval, write the stats to
1736  * $DATADIR/stats/served-desc-stats (possibly appending to an existing file)
1737  * and reset the state for the next interval. Return when we would next want
1738  * to write served desc stats or 0 if we won't want to write. */
1739 time_t
1741 {
1742  char *filename = NULL, *str = NULL;
1743 
1745  return 0; /* We're not collecting stats. */
1746  if (start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL > now)
1747  return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL;
1748 
1749  str = rep_hist_format_desc_stats(now);
1750  tor_assert(str != NULL);
1751 
1752  if (check_or_create_data_subdir("stats") < 0) {
1753  goto done;
1754  }
1755  filename = get_datadir_fname2("stats", "served-desc-stats");
1756  if (append_bytes_to_file(filename, str, strlen(str), 0) < 0)
1757  log_warn(LD_HIST, "Unable to write served descs statistics to disk!");
1758 
1760 
1761  done:
1762  tor_free(filename);
1763  tor_free(str);
1764  return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL;
1765 }
1766 
1767 /** Called to note that we've served a given descriptor (by
1768  * digest). Increments the count of descriptors served, and the number
1769  * of times we've served this descriptor. */
1770 void
1771 rep_hist_note_desc_served(const char * desc)
1772 {
1773  void *val;
1774  uintptr_t count;
1775  if (!served_descs)
1776  return; // We're not collecting stats
1777  val = digestmap_get(served_descs, desc);
1778  count = (uintptr_t)val;
1779  if (count != INT_MAX)
1780  ++count;
1781  digestmap_set(served_descs, desc, (void*)count);
1783 }
1784 
1785 /*** Connection statistics ***/
1786 
1787 /** Internal statistics to track how many requests of each type of
1788  * handshake we've received, and how many we've assigned to cpuworkers.
1789  * Useful for seeing trends in cpu load.
1790  * @{ */
1791 STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0};
1792 STATIC int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0};
1793 /**@}*/
1794 
1795 /** A new onionskin (using the <b>type</b> handshake) has arrived. */
1796 void
1798 {
1799  if (type <= MAX_ONION_HANDSHAKE_TYPE)
1801 }
1802 
1803 /** We've sent an onionskin (using the <b>type</b> handshake) to a
1804  * cpuworker. */
1805 void
1807 {
1808  if (type <= MAX_ONION_HANDSHAKE_TYPE)
1809  onion_handshakes_assigned[type]++;
1810 }
1811 
1812 /** Get the circuit handshake value that is requested. */
1813 MOCK_IMPL(int,
1815 {
1816  if (BUG(type > MAX_ONION_HANDSHAKE_TYPE)) {
1817  return 0;
1818  }
1819  return onion_handshakes_requested[type];
1820 }
1821 
1822 /** Get the circuit handshake value that is assigned. */
1823 MOCK_IMPL(int,
1825 {
1826  if (BUG(type > MAX_ONION_HANDSHAKE_TYPE)) {
1827  return 0;
1828  }
1829  return onion_handshakes_assigned[type];
1830 }
1831 
1832 /** Log our onionskin statistics since the last time we were called. */
1833 void
1835 {
1836  (void)now;
1837  log_notice(LD_HEARTBEAT, "Circuit handshake stats since last time: "
1838  "%d/%d TAP, %d/%d NTor.",
1839  onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_TAP],
1840  onion_handshakes_requested[ONION_HANDSHAKE_TYPE_TAP],
1841  onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_NTOR],
1842  onion_handshakes_requested[ONION_HANDSHAKE_TYPE_NTOR]);
1843  memset(onion_handshakes_assigned, 0, sizeof(onion_handshakes_assigned));
1845 }
1846 
1847 /* Hidden service statistics section */
1848 
1849 /** Start of the current hidden service stats interval or 0 if we're
1850  * not collecting hidden service statistics. */
1852 
1853 /** Our v2 statistics structure singleton. */
1854 static hs_v2_stats_t *hs_v2_stats = NULL;
1855 
1856 /** HSv2 stats */
1857 
1858 /** Allocate, initialize and return an hs_v2_stats_t structure. */
1859 static hs_v2_stats_t *
1861 {
1862  hs_v2_stats_t *new_hs_v2_stats = tor_malloc_zero(sizeof(hs_v2_stats_t));
1863 
1864  return new_hs_v2_stats;
1865 }
1866 
1867 #define hs_v2_stats_free(val) \
1868  FREE_AND_NULL(hs_v2_stats_t, hs_v2_stats_free_, (val))
1869 
1870 /** Free an hs_v2_stats_t structure. */
1871 static void
1872 hs_v2_stats_free_(hs_v2_stats_t *victim_hs_v2_stats)
1873 {
1874  if (!victim_hs_v2_stats) {
1875  return;
1876  }
1877  tor_free(victim_hs_v2_stats);
1878 }
1879 
1880 /** Clear history of hidden service statistics and set the measurement
1881  * interval start to <b>now</b>. */
1882 static void
1884 {
1885  if (!hs_v2_stats) {
1887  }
1888 
1889  hs_v2_stats->rp_v2_relay_cells_seen = 0;
1890 
1892 }
1893 
1894 /*** HSv3 stats ******/
1895 
1896 /** Start of the current hidden service stats interval or 0 if we're not
1897  * collecting hidden service statistics.
1898  *
1899  * This is particularly important for v3 statistics since this variable
1900  * controls the start time of initial v3 stats collection. It's initialized by
1901  * rep_hist_hs_stats_init() to the next time period start (i.e. 12:00UTC), and
1902  * should_collect_v3_stats() ensures that functions that collect v3 stats do
1903  * not do so sooner than that.
1904  *
1905  * Collecting stats from 12:00UTC to 12:00UTC is extremely important for v3
1906  * stats because rep_hist_hsdir_stored_maybe_new_v3_onion() uses the blinded
1907  * key of each onion service as its double-counting index. Onion services
1908  * rotate their descriptor at around 00:00UTC which means that their blinded
1909  * key also changes around that time. However the precise time that onion
1910  * services rotate their descriptors is actually when they fetch a new
1911  * 00:00UTC consensus and that happens at a random time (e.g. it can even
1912  * happen at 02:00UTC). This means that if we started keeping v3 stats at
1913  * around 00:00UTC we wouldn't be able to tell when onion services change
1914  * their blinded key and hence we would double count an unpredictable amount
1915  * of them (for example, if an onion service fetches the 00:00UTC consensus at
1916  * 01:00UTC it would upload to its old HSDir at 00:45UTC, and then to a
1917  * different HSDir at 01:50UTC).
1918  *
1919  * For this reason, we start collecting statistics at 12:00UTC. This way we
1920  * know that by the time we stop collecting statistics for that time period 24
1921  * hours later, all the onion services have switched to their new blinded
1922  * key. This way we can predict much better how much double counting has been
1923  * performed.
1924  */
1926 
1927 /** Our v3 statistics structure singleton. */
1928 static hs_v3_stats_t *hs_v3_stats = NULL;
1929 
1930 /** Allocate, initialize and return an hs_v3_stats_t structure. */
1931 static hs_v3_stats_t *
1933 {
1934  hs_v3_stats_t *new_hs_v3_stats = tor_malloc_zero(sizeof(hs_v3_stats_t));
1935  new_hs_v3_stats->v3_onions_seen_this_period = digest256map_new();
1936 
1937  return new_hs_v3_stats;
1938 }
1939 
1940 #define hs_v3_stats_free(val) \
1941  FREE_AND_NULL(hs_v3_stats_t, hs_v3_stats_free_, (val))
1942 
1943 /** Free an hs_v3_stats_t structure. */
1944 static void
1945 hs_v3_stats_free_(hs_v3_stats_t *victim_hs_v3_stats)
1946 {
1947  if (!victim_hs_v3_stats) {
1948  return;
1949  }
1950 
1951  digest256map_free(victim_hs_v3_stats->v3_onions_seen_this_period, NULL);
1952  tor_free(victim_hs_v3_stats);
1953 }
1954 
1955 /** Clear history of hidden service statistics and set the measurement
1956  * interval start to <b>now</b>. */
1957 static void
1959 {
1960  if (!hs_v3_stats) {
1962  }
1963 
1964  digest256map_free(hs_v3_stats->v3_onions_seen_this_period, NULL);
1965  hs_v3_stats->v3_onions_seen_this_period = digest256map_new();
1966 
1967  hs_v3_stats->rp_v3_relay_cells_seen = 0;
1968 
1970 }
1971 
1972 /** Return true if it's a good time to collect v3 stats.
1973  *
1974  * v3 stats have a strict stats collection period (from 12:00UTC to 12:00UTC
1975  * on the real network). We don't want to collect statistics if (for example)
1976  * we just booted and it's 03:00UTC; we will wait until 12:00UTC before we
1977  * start collecting statistics to make sure that the final result represents
1978  * the whole collection period. This behavior is controlled by
1979  * rep_hist_hs_stats_init().
1980  */
1981 MOCK_IMPL(STATIC bool,
1983 {
1985 }
1986 
1987 /** We just received a new descriptor with <b>blinded_key</b>. See if we've
1988  * seen this blinded key before, and if not add it to the stats. */
1989 void
1990 rep_hist_hsdir_stored_maybe_new_v3_onion(const uint8_t *blinded_key)
1991 {
1992  /* Return early if we don't collect HSv3 stats, or if it's not yet the time
1993  * to collect them. */
1994  if (!hs_v3_stats || !should_collect_v3_stats()) {
1995  return;
1996  }
1997 
1998  bool seen_before =
1999  !!digest256map_get(hs_v3_stats->v3_onions_seen_this_period,
2000  blinded_key);
2001 
2002  log_info(LD_GENERAL, "Considering v3 descriptor with %s (%sseen before)",
2003  safe_str(hex_str((char*)blinded_key, 32)),
2004  seen_before ? "" : "not ");
2005 
2006  /* Count it if we haven't seen it before. */
2007  if (!seen_before) {
2008  digest256map_set(hs_v3_stats->v3_onions_seen_this_period,
2009  blinded_key, (void*)(uintptr_t)1);
2010  }
2011 }
2012 
2013 /** We saw a new HS relay cell: count it!
2014  * If <b>is_v2</b> is set then it's a v2 RP cell, otherwise it's a v3. */
2015 void
2017 {
2018  log_debug(LD_GENERAL, "New RP cell (%d)", is_v2);
2019 
2020  if (is_v2 && hs_v2_stats) {
2021  hs_v2_stats->rp_v2_relay_cells_seen++;
2022  } else if (!is_v2 && hs_v3_stats && should_collect_v3_stats()) {
2023  hs_v3_stats->rp_v3_relay_cells_seen++;
2024  }
2025 }
2026 
2027 /** Generic HS stats code */
2028 
2029 /** Initialize v2 and v3 hidden service statistics. */
2030 void
2032 {
2033  if (!hs_v2_stats) {
2035  }
2036 
2037  /* Start collecting v2 stats straight away */
2039 
2040  if (!hs_v3_stats) {
2042  }
2043 
2044  /* Start collecting v3 stats at the next 12:00 UTC */
2046 }
2047 
2048 /** Stop collecting hidden service stats in a way that we can re-start
2049  * doing so in rep_hist_buffer_stats_init(). */
2050 void
2052 {
2055 }
2056 
2057 /** Stats reporting code */
2058 
2059 /* The number of cells that are supposed to be hidden from the adversary
2060  * by adding noise from the Laplace distribution. This value, divided by
2061  * EPSILON, is Laplace parameter b. It must be greater than 0. */
2062 #define REND_CELLS_DELTA_F 2048
2063 /* Security parameter for obfuscating number of cells with a value between
2064  * ]0.0, 1.0]. Smaller values obfuscate observations more, but at the same
2065  * time make statistics less usable. */
2066 #define REND_CELLS_EPSILON 0.3
2067 /* The number of cells that are supposed to be hidden from the adversary
2068  * by rounding up to the next multiple of this number. */
2069 #define REND_CELLS_BIN_SIZE 1024
2070 /* The number of service identities that are supposed to be hidden from the
2071  * adversary by adding noise from the Laplace distribution. This value,
2072  * divided by EPSILON, is Laplace parameter b. It must be greater than 0. */
2073 #define ONIONS_SEEN_DELTA_F 8
2074 /* Security parameter for obfuscating number of service identities with a
2075  * value between ]0.0, 1.0]. Smaller values obfuscate observations more, but
2076  * at the same time make statistics less usable. */
2077 #define ONIONS_SEEN_EPSILON 0.3
2078 /* The number of service identities that are supposed to be hidden from
2079  * the adversary by rounding up to the next multiple of this number. */
2080 #define ONIONS_SEEN_BIN_SIZE 8
2081 
2082 /** Allocate and return a string containing hidden service stats that
2083  * are meant to be placed in the extra-info descriptor.
2084  *
2085  * Function works for both v2 and v3 stats depending on <b>is_v3</b>. */
2086 STATIC char *
2087 rep_hist_format_hs_stats(time_t now, bool is_v3)
2088 {
2089  char t[ISO_TIME_LEN+1];
2090  char *hs_stats_string;
2091  int64_t obfuscated_onions_seen, obfuscated_cells_seen;
2092 
2093  uint64_t rp_cells_seen = is_v3 ?
2094  hs_v3_stats->rp_v3_relay_cells_seen : hs_v2_stats->rp_v2_relay_cells_seen;
2095  size_t onions_seen = is_v3 ?
2096  digest256map_size(hs_v3_stats->v3_onions_seen_this_period) : 0;
2097  time_t start_of_hs_stats_interval = is_v3 ?
2099 
2100  uint64_t rounded_cells_seen
2101  = round_uint64_to_next_multiple_of(rp_cells_seen, REND_CELLS_BIN_SIZE);
2102  rounded_cells_seen = MIN(rounded_cells_seen, INT64_MAX);
2103  obfuscated_cells_seen = add_laplace_noise((int64_t)rounded_cells_seen,
2105  REND_CELLS_DELTA_F, REND_CELLS_EPSILON);
2106 
2107  uint64_t rounded_onions_seen =
2108  round_uint64_to_next_multiple_of(onions_seen, ONIONS_SEEN_BIN_SIZE);
2109  rounded_onions_seen = MIN(rounded_onions_seen, INT64_MAX);
2110  obfuscated_onions_seen = add_laplace_noise((int64_t)rounded_onions_seen,
2111  crypto_rand_double(), ONIONS_SEEN_DELTA_F,
2112  ONIONS_SEEN_EPSILON);
2113 
2114  format_iso_time(t, now);
2115  tor_asprintf(&hs_stats_string, "%s %s (%u s)\n"
2116  "%s %"PRId64" delta_f=%d epsilon=%.2f bin_size=%d\n"
2117  "%s %"PRId64" delta_f=%d epsilon=%.2f bin_size=%d\n",
2118  is_v3 ? "hidserv-v3-stats-end" : "hidserv-stats-end",
2119  t, (unsigned) (now - start_of_hs_stats_interval),
2120  is_v3 ?
2121  "hidserv-rend-v3-relayed-cells" : "hidserv-rend-relayed-cells",
2122  obfuscated_cells_seen, REND_CELLS_DELTA_F,
2123  REND_CELLS_EPSILON, REND_CELLS_BIN_SIZE,
2124  is_v3 ? "hidserv-dir-v3-onions-seen" :"hidserv-dir-onions-seen",
2125  obfuscated_onions_seen, ONIONS_SEEN_DELTA_F,
2126  ONIONS_SEEN_EPSILON, ONIONS_SEEN_BIN_SIZE);
2127 
2128  return hs_stats_string;
2129 }
2130 
2131 /** If 24 hours have passed since the beginning of the current HS
2132  * stats period, write buffer stats to $DATADIR/stats/hidserv-v3-stats
2133  * (possibly overwriting an existing file) and reset counters. Return
2134  * when we would next want to write buffer stats or 0 if we never want to
2135  * write. Function works for both v2 and v3 stats depending on <b>is_v3</b>.
2136  */
2137 time_t
2138 rep_hist_hs_stats_write(time_t now, bool is_v3)
2139 {
2140  char *str = NULL;
2141 
2142  time_t start_of_hs_stats_interval = is_v3 ?
2144 
2145  if (!start_of_hs_stats_interval) {
2146  return 0; /* Not initialized. */
2147  }
2148 
2149  if (start_of_hs_stats_interval + WRITE_STATS_INTERVAL > now) {
2150  goto done; /* Not ready to write */
2151  }
2152 
2153  /* Generate history string. */
2154  str = rep_hist_format_hs_stats(now, is_v3);
2155 
2156  /* Reset HS history. */
2157  if (is_v3) {
2159  } else {
2161  }
2162 
2163  /* Try to write to disk. */
2164  if (!check_or_create_data_subdir("stats")) {
2165  write_to_data_subdir("stats",
2166  is_v3 ? "hidserv-v3-stats" : "hidserv-stats",
2167  str, "hidden service stats");
2168  }
2169 
2170  done:
2171  tor_free(str);
2172  return start_of_hs_stats_interval + WRITE_STATS_INTERVAL;
2173 }
2174 
2175 static uint64_t link_proto_count[MAX_LINK_PROTO+1][2];
2176 
2177 /** Note that we negotiated link protocol version <b>link_proto</b>, on
2178  * a connection that started here iff <b>started_here</b> is true.
2179  */
2180 void
2181 rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
2182 {
2183  started_here = !!started_here; /* force to 0 or 1 */
2184  if (link_proto > MAX_LINK_PROTO) {
2185  log_warn(LD_BUG, "Can't log link protocol %u", link_proto);
2186  return;
2187  }
2188 
2189  link_proto_count[link_proto][started_here]++;
2190 }
2191 
2192 /**
2193  * Update the maximum count of total pending channel padding timers
2194  * in this period.
2195  */
2196 void
2197 rep_hist_padding_count_timers(uint64_t num_timers)
2198 {
2199  if (num_timers > padding_current.maximum_chanpad_timers) {
2201  }
2202 }
2203 
2204 /**
2205  * Count a cell that we sent for padding overhead statistics.
2206  *
2207  * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
2208  * counted for PADDING_TYPE_TOTAL.
2209  */
2210 void
2212 {
2213  switch (type) {
2214  case PADDING_TYPE_DROP:
2216  break;
2217  case PADDING_TYPE_CELL:
2219  break;
2220  case PADDING_TYPE_TOTAL:
2222  break;
2225  break;
2228  break;
2229  }
2230 }
2231 
2232 /**
2233  * Count a cell that we've received for padding overhead statistics.
2234  *
2235  * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
2236  * counted for PADDING_TYPE_TOTAL.
2237  */
2238 void
2240 {
2241  switch (type) {
2242  case PADDING_TYPE_DROP:
2244  break;
2245  case PADDING_TYPE_CELL:
2247  break;
2248  case PADDING_TYPE_TOTAL:
2250  break;
2253  break;
2256  break;
2257  }
2258 }
2259 
2260 /**
2261  * Reset our current padding statistics. Called once every 24 hours.
2262  */
2263 void
2265 {
2266  memset(&padding_current, 0, sizeof(padding_current));
2267 }
2268 
2269 /**
2270  * Copy our current cell counts into a structure for listing in our
2271  * extra-info descriptor. Also perform appropriate rounding and redaction.
2272  *
2273  * This function is called once every 24 hours.
2274  */
2275 #define MIN_CELL_COUNTS_TO_PUBLISH 1
2276 #define ROUND_CELL_COUNTS_TO 10000
2277 void
2278 rep_hist_prep_published_padding_counts(time_t now)
2279 {
2281 
2284  memset(&padding_published, 0, sizeof(padding_published));
2285  return;
2286  }
2287 
2289 #define ROUND_AND_SET_COUNT(x) (x) = round_uint64_to_next_multiple_of((x), \
2290  ROUND_CELL_COUNTS_TO)
2291  ROUND_AND_SET_COUNT(padding_published.read_pad_cell_count);
2292  ROUND_AND_SET_COUNT(padding_published.write_pad_cell_count);
2293  ROUND_AND_SET_COUNT(padding_published.read_drop_cell_count);
2294  ROUND_AND_SET_COUNT(padding_published.write_drop_cell_count);
2295  ROUND_AND_SET_COUNT(padding_published.write_cell_count);
2296  ROUND_AND_SET_COUNT(padding_published.read_cell_count);
2297  ROUND_AND_SET_COUNT(padding_published.enabled_read_cell_count);
2298  ROUND_AND_SET_COUNT(padding_published.enabled_read_pad_cell_count);
2299  ROUND_AND_SET_COUNT(padding_published.enabled_write_cell_count);
2300  ROUND_AND_SET_COUNT(padding_published.enabled_write_pad_cell_count);
2301 #undef ROUND_AND_SET_COUNT
2302 }
2303 
2304 /**
2305  * Returns an allocated string for extra-info documents for publishing
2306  * padding statistics from the last 24 hour interval.
2307  */
2308 char *
2310 {
2311  char *result = NULL;
2312 
2315  return NULL;
2316  }
2317 
2318  tor_asprintf(&result, "padding-counts %s (%d s)"
2319  " bin-size=%"PRIu64
2320  " write-drop=%"PRIu64
2321  " write-pad=%"PRIu64
2322  " write-total=%"PRIu64
2323  " read-drop=%"PRIu64
2324  " read-pad=%"PRIu64
2325  " read-total=%"PRIu64
2326  " enabled-read-pad=%"PRIu64
2327  " enabled-read-total=%"PRIu64
2328  " enabled-write-pad=%"PRIu64
2329  " enabled-write-total=%"PRIu64
2330  " max-chanpad-timers=%"PRIu64
2331  "\n",
2334  (uint64_t)ROUND_CELL_COUNTS_TO,
2346  );
2347 
2348  return result;
2349 }
2350 
2351 /** Log a heartbeat message explaining how many connections of each link
2352  * protocol version we have used.
2353  */
2354 void
2356 {
2357  smartlist_t *lines = smartlist_new();
2358 
2359  for (int i = 1; i <= MAX_LINK_PROTO; i++) {
2360  char *line = NULL;
2361  tor_asprintf(&line, "initiated %"PRIu64" and received "
2362  "%"PRIu64" v%d connections", link_proto_count[i][1],
2363  link_proto_count[i][0], i);
2364  smartlist_add(lines, line);
2365  }
2366 
2367  char *log_line = smartlist_join_strings(lines, "; ", 0, NULL);
2368 
2369  log_notice(LD_HEARTBEAT, "Since startup we %s.", log_line);
2370 
2371  SMARTLIST_FOREACH(lines, char *, s, tor_free(s));
2372  smartlist_free(lines);
2373  tor_free(log_line);
2374 }
2375 
2376 /** Free all storage held by the OR/link history caches, by the
2377  * bandwidth history arrays, by the port history, or by statistics . */
2378 void
2380 {
2381  hs_v2_stats_free(hs_v2_stats);
2382  hs_v3_stats_free(hs_v3_stats);
2383  digestmap_free(history_map, free_or_history);
2384 
2390 
2393  tor_free(s));
2394  smartlist_free(circuits_for_buffer_stats);
2396  }
2399 
2400  tor_assert_nonfatal(rephist_total_alloc == 0);
2401  tor_assert_nonfatal_once(rephist_total_num == 0);
2402 }
2403 
2404 #ifdef TOR_UNIT_TESTS
2405 /* only exists for unit tests: get HSv2 stats object */
2406 const hs_v2_stats_t *
2407 rep_hist_get_hs_v2_stats(void)
2408 {
2409  return hs_v2_stats;
2410 }
2411 
2412 /* only exists for unit tests: get HSv2 stats object */
2413 const hs_v3_stats_t *
2414 rep_hist_get_hs_v3_stats(void)
2415 {
2416  return hs_v3_stats;
2417 }
2418 #endif /* defined(TOR_UNIT_TESTS) */
finish_writing_to_file
int finish_writing_to_file(open_file_t *file_data)
Definition: files.c:465
rep_hist_downrate_old_runs
time_t rep_hist_downrate_old_runs(time_t now)
Definition: rephist.c:507
hs_v3_stats_free_
static void hs_v3_stats_free_(hs_v3_stats_t *victim_hs_v3_stats)
Definition: rephist.c:1945
padding_counts_t::first_published_at
char first_published_at[ISO_TIME_LEN+1]
Definition: rephist.c:165
tor_addr_compare
int tor_addr_compare(const tor_addr_t *addr1, const tor_addr_t *addr2, tor_addr_comparison_t how)
Definition: address.c:984
tor_free
#define tor_free(p)
Definition: malloc.h:52
or_history_t::weighted_run_length
unsigned long weighted_run_length
Definition: rephist.c:126
padding_counts_t::write_pad_cell_count
uint64_t write_pad_cell_count
Definition: rephist.c:149
networkstatus_t::valid_until
time_t valid_until
Definition: networkstatus_st.h:36
rep_hist_format_desc_stats
static char * rep_hist_format_desc_stats(time_t now)
Definition: rephist.c:1686
smartlist_split_string
int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep, int flags, int max)
Definition: smartlist_split.c:37
hex_str
const char * hex_str(const char *from, size_t fromlen)
Definition: binascii.c:34
or_options_t::BandwidthRate
uint64_t BandwidthRate
Definition: or_options_st.h:416
start_of_hs_v3_stats_interval
static time_t start_of_hs_v3_stats_interval
Definition: rephist.c:1925
tor_addr_t
Definition: address.h:69
STABILITY_EPSILON
#define STABILITY_EPSILON
Definition: rephist.c:98
or_history_t::last_reached_addr
tor_addr_t last_reached_addr
Definition: rephist.c:118
stability_last_downrated
static time_t stability_last_downrated
Definition: rephist.c:178
approx_time
time_t approx_time(void)
Definition: approx_time.c:32
rep_hist_get_uptime
long rep_hist_get_uptime(const char *id, time_t when)
Definition: rephist.c:615
MOCK_IMPL
#define MOCK_IMPL(rv, funcname, arglist)
Definition: testsupport.h:133
circuit_get_global_list
smartlist_t * circuit_get_global_list(void)
Definition: circuitlist.c:694
hs_v2_stats
static hs_v2_stats_t * hs_v2_stats
Definition: rephist.c:1854
add_laplace_noise
int64_t add_laplace_noise(int64_t signal_, double random_, double delta_f, double epsilon)
Definition: laplace.c:51
rep_hist_load_mtbf_data
int rep_hist_load_mtbf_data(time_t now)
Definition: rephist.c:914
circ_buffer_stats_t::mean_num_cells_in_queue
double mean_num_cells_in_queue
Definition: rephist.c:1395
get_weighted_fractional_uptime
static double get_weighted_fractional_uptime(or_history_t *hist, time_t when)
Definition: rephist.c:588
tor_assert
#define tor_assert(expr)
Definition: util_bug.h:102
circ_buffer_stats_t::processed_cells
uint32_t processed_cells
Definition: rephist.c:1399
LD_BUG
#define LD_BUG
Definition: log.h:86
get_stability
static double get_stability(or_history_t *hist, time_t when)
Definition: rephist.c:552
rep_hist_note_overload
void rep_hist_note_overload(overload_type_t overload)
Definition: rephist.c:292
or_history_t
Definition: rephist.c:110
rep_hist_reset_exit_stats
void rep_hist_reset_exit_stats(time_t now)
Definition: rephist.c:1139
format_iso_time
void format_iso_time(char *buf, time_t t)
Definition: time_fmt.c:295
append_bytes_to_file
int append_bytes_to_file(const char *fname, const char *str, size_t len, int bin)
Definition: files.c:554
compare_int_
static int compare_int_(const void *x, const void *y)
Definition: rephist.c:1162
LD_GENERAL
#define LD_GENERAL
Definition: log.h:62
PADDING_TYPE_ENABLED_CELL
@ PADDING_TYPE_ENABLED_CELL
Definition: rephist.h:131
rep_hist_free_all
void rep_hist_free_all(void)
Definition: rephist.c:2379
base16_encode
void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:478
should_collect_v3_stats
STATIC bool should_collect_v3_stats(void)
Definition: rephist.c:1982
rep_hist_desc_stats_init
void rep_hist_desc_stats_init(time_t now)
Definition: rephist.c:1650
padding_counts_t
Definition: rephist.c:141
rep_hist_padding_count_timers
void rep_hist_padding_count_timers(uint64_t num_timers)
Definition: rephist.c:2197
EXIT_STATS_NUM_PORTS
#define EXIT_STATS_NUM_PORTS
Definition: rephist.c:1109
rep_hist_exit_stats_term
void rep_hist_exit_stats_term(void)
Definition: rephist.c:1150
rep_hist_note_exit_stream_opened
void rep_hist_note_exit_stream_opened(uint16_t port)
Definition: rephist.c:1370
node_get_by_id
const node_t * node_get_by_id(const char *identity_digest)
Definition: nodelist.c:226
smartlist_add
void smartlist_add(smartlist_t *sl, void *element)
Definition: smartlist_core.c:117
get_total_weighted_time
static long get_total_weighted_time(or_history_t *hist, time_t when)
Definition: rephist.c:574
rep_hist_have_measured_enough_stability
int rep_hist_have_measured_enough_stability(void)
Definition: rephist.c:668
circ_buffer_stats_t::mean_time_cells_in_queue
double mean_time_cells_in_queue
Definition: rephist.c:1397
overload_stats_t
Definition: rephist.c:190
check_or_create_data_subdir
int check_or_create_data_subdir(const char *subdir)
Definition: config.c:7046
rep_hist_exit_stats_init
void rep_hist_exit_stats_init(time_t now)
Definition: rephist.c:1129
onion_handshakes_requested
STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1]
Definition: rephist.c:1791
rep_hist_note_desc_served
void rep_hist_note_desc_served(const char *desc)
Definition: rephist.c:1771
EXIT_STATS_ROUND_UP_BYTES
#define EXIT_STATS_ROUND_UP_BYTES
Definition: rephist.c:1105
LD_HIST
#define LD_HIST
Definition: log.h:99
rep_hist_get_overload_stats_lines
char * rep_hist_get_overload_stats_lines(void)
Definition: rephist.c:245
hs_v3_stats_new
static hs_v3_stats_t * hs_v3_stats_new(void)
Definition: rephist.c:1932
exit_streams
static uint32_t * exit_streams
Definition: rephist.c:1122
smartlist_new
smartlist_t * smartlist_new(void)
Definition: smartlist_core.c:26
tor_addr_make_unspec
void tor_addr_make_unspec(tor_addr_t *a)
Definition: address.c:225
round_uint64_to_next_multiple_of
uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor)
Definition: muldiv.c:50
CIRCUIT_IS_ORIGIN
#define CIRCUIT_IS_ORIGIN(c)
Definition: circuitlist.h:147
rep_hist_note_router_reachable
void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, const uint16_t at_port, time_t when)
Definition: rephist.c:366
served_descs
static digestmap_t * served_descs
Definition: rephist.c:1639
tor_parse_long
long tor_parse_long(const char *s, int base, long min, long max, int *ok, char **next)
Definition: parse_int.c:59
open_file_t
Definition: files.c:289
REND_CELLS_DELTA_F
#define REND_CELLS_DELTA_F
Definition: rephist.c:2062
parse_iso_time
int parse_iso_time(const char *cp, time_t *t)
Definition: time_fmt.c:392
EXIT_STATS_ROUND_UP_STREAMS
#define EXIT_STATS_ROUND_UP_STREAMS
Definition: rephist.c:1107
SMARTLIST_FOREACH
#define SMARTLIST_FOREACH(sl, type, var, cmd)
Definition: smartlist_foreach.h:112
rep_hist_buffer_stats_init
void rep_hist_buffer_stats_init(time_t now)
Definition: rephist.c:1386
rep_hist_reset_hs_v3_stats
static void rep_hist_reset_hs_v3_stats(time_t now)
Definition: rephist.c:1958
overload_type_t
overload_type_t
Definition: rephist.h:147
networkstatus.h
Header file for networkstatus.c.
n_bogus_times
static int n_bogus_times
Definition: rephist.c:869
STABILITY_INTERVAL
#define STABILITY_INTERVAL
Definition: rephist.c:103
rep_hist_note_negotiated_link_proto
void rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
Definition: rephist.c:2181
rep_hist_buffer_stats_add_circ
void rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval)
Definition: rephist.c:1428
rep_history_clean
void rep_history_clean(time_t before)
Definition: rephist.c:718
rep_hist_padding_count_read
void rep_hist_padding_count_read(padding_type_t type)
Definition: rephist.c:2239
base16_decode
int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:506
rep_hist_dump_stats
void rep_hist_dump_stats(time_t now, int severity)
Definition: rephist.c:679
REPHIST_CELL_PADDING_COUNTS_INTERVAL
#define REPHIST_CELL_PADDING_COUNTS_INTERVAL
Definition: rephist.h:135
padding_counts_t::maximum_chanpad_timers
uint64_t maximum_chanpad_timers
Definition: rephist.c:163
rep_hist_buffer_stats_term
void rep_hist_buffer_stats_term(void)
Definition: rephist.c:1478
circuitlist.h
Header file for circuitlist.c.
DIGEST_LEN
#define DIGEST_LEN
Definition: digest_sizes.h:20
get_or_history
static or_history_t * get_or_history(const char *id)
Definition: rephist.c:327
rep_hist_desc_stats_term
void rep_hist_desc_stats_term(void)
Definition: rephist.c:1673
rep_hist_add_buffer_stats
void rep_hist_add_buffer_stats(double mean_num_cells_in_queue, double mean_time_cells_in_queue, uint32_t processed_cells)
Definition: rephist.c:1409
rep_hist_reset_padding_counts
void rep_hist_reset_padding_counts(void)
Definition: rephist.c:2264
or_history_t::total_run_weights
double total_run_weights
Definition: rephist.c:131
rep_hist_reset_buffer_stats
void rep_hist_reset_buffer_stats(time_t now)
Definition: rephist.c:1486
rep_hist_get_weighted_time_known
long rep_hist_get_weighted_time_known(const char *id, time_t when)
Definition: rephist.c:656
PADDING_TYPE_ENABLED_TOTAL
@ PADDING_TYPE_ENABLED_TOTAL
Definition: rephist.h:129
padding_counts_t::read_pad_cell_count
uint64_t read_pad_cell_count
Definition: rephist.c:147
predicted_ports_free_all
void predicted_ports_free_all(void)
Definition: predict_ports.c:302
hs_v2_stats_free_
static void hs_v2_stats_free_(hs_v2_stats_t *victim_hs_v2_stats)
Definition: rephist.c:1872
rep_hist_log_link_protocol_counts
void rep_hist_log_link_protocol_counts(void)
Definition: rephist.c:2355
networkstatus_get_latest_consensus
networkstatus_t * networkstatus_get_latest_consensus(void)
Definition: networkstatus.c:1390
padding_counts_t::enabled_write_pad_cell_count
uint64_t enabled_write_pad_cell_count
Definition: rephist.c:157
node_t
Definition: node_st.h:34
rep_hist_init
void rep_hist_init(void)
Definition: rephist.c:358
RFTS_IGNORE_MISSING
#define RFTS_IGNORE_MISSING
Definition: files.h:101
rep_hist_get_stability
double rep_hist_get_stability(const char *id, time_t when)
Definition: rephist.c:628
padding_type_t
padding_type_t
Definition: rephist.h:121
rephist_total_num
uint32_t rephist_total_num
Definition: rephist.c:94
rep_hist_hs_stats_term
void rep_hist_hs_stats_term(void)
Definition: rephist.c:2051
rep_hist_hs_stats_init
void rep_hist_hs_stats_init(time_t now)
Definition: rephist.c:2031
buffer_stats_compare_entries_
static int buffer_stats_compare_entries_(const void **_a, const void **_b)
Definition: rephist.c:1464
tor_digest_is_zero
int tor_digest_is_zero(const char *digest)
Definition: util_string.c:96
hs_v2_stats_new
static hs_v2_stats_t * hs_v2_stats_new(void)
Definition: rephist.c:1860
exit_bytes_read
static uint64_t * exit_bytes_read
Definition: rephist.c:1118
padding_counts_t::write_drop_cell_count
uint64_t write_drop_cell_count
Definition: rephist.c:161
escaped
const char * escaped(const char *s)
Definition: escape.c:126
strcmpstart
int strcmpstart(const char *s1, const char *s2)
Definition: util_string.c:215
find_next_with
static int find_next_with(smartlist_t *sl, int i, const char *prefix)
Definition: rephist.c:856
SET_TO_START_OF_HOUR
#define SET_TO_START_OF_HOUR(a)
Definition: rephist.c:286
authmode.h
Header file for directory authority mode.
circuits_for_buffer_stats
static smartlist_t * circuits_for_buffer_stats
Definition: rephist.c:1403
circuit_t
Definition: circuit_st.h:61
nodelist.h
Header file for nodelist.c.
LD_HEARTBEAT
#define LD_HEARTBEAT
Definition: log.h:103
rep_hist_get_weighted_fractional_uptime
double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when)
Definition: rephist.c:640
laplace.h
Header for laplace.c.
rep_hist_format_buffer_stats
char * rep_hist_format_buffer_stats(time_t now)
Definition: rephist.c:1500
predict_ports.h
Header file for predict_ports.c.
rep_hist_get_circuit_handshake_assigned
int rep_hist_get_circuit_handshake_assigned(uint16_t type)
Definition: rephist.c:1824
conn_stats_free_all
void conn_stats_free_all(void)
Definition: connstats.c:112
circuit_t::timestamp_created
struct timeval timestamp_created
Definition: circuit_st.h:168
abort_writing_to_file
int abort_writing_to_file(open_file_t *file_data)
Definition: files.c:473
EXIT_STATS_TOP_N_PORTS
#define EXIT_STATS_TOP_N_PORTS
Definition: rephist.c:1111
rep_hist_get_overload_general_line
char * rep_hist_get_overload_general_line(void)
Definition: rephist.c:227
padding_counts_t::read_drop_cell_count
uint64_t read_drop_cell_count
Definition: rephist.c:159
round_uint32_to_next_multiple_of
uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor)
Definition: muldiv.c:35
tor_sscanf
int tor_sscanf(const char *buf, const char *pattern,...)
Definition: scanf.c:309
PADDING_TYPE_TOTAL
@ PADDING_TYPE_TOTAL
Definition: rephist.h:127
rep_hist_buffer_stats_write
time_t rep_hist_buffer_stats_write(time_t now)
Definition: rephist.c:1603
STABILITY_ALPHA
#define STABILITY_ALPHA
Definition: rephist.c:101
rep_hist_padding_count_write
void rep_hist_padding_count_write(padding_type_t type)
Definition: rephist.c:2211
parse_possibly_bad_iso_time
static int parse_possibly_bad_iso_time(const char *s, time_t *time_out)
Definition: rephist.c:873
rep_hist_get_circuit_handshake_requested
int rep_hist_get_circuit_handshake_requested(uint16_t type)
Definition: rephist.c:1814
crypto_rand.h
Common functions for using (pseudo-)random number generators.
get_options
const or_options_t * get_options(void)
Definition: config.c:919
rep_hist_note_circuit_handshake_requested
void rep_hist_note_circuit_handshake_requested(uint16_t type)
Definition: rephist.c:1797
padding_current
static padding_counts_t padding_current
Definition: rephist.c:170
rep_hist_note_router_unreachable
void rep_hist_note_router_unreachable(const char *id, time_t when)
Definition: rephist.c:439
rep_hist_make_router_pessimal
void rep_hist_make_router_pessimal(const char *id, time_t when)
Definition: rephist.c:493
write_to_data_subdir
int write_to_data_subdir(const char *subdir, const char *fname, const char *str, const char *descr)
Definition: config.c:7065
padding_counts_t::enabled_write_cell_count
uint64_t enabled_write_cell_count
Definition: rephist.c:153
rep_hist_format_hs_stats
STATIC char * rep_hist_format_hs_stats(time_t now, bool is_v3)
Definition: rephist.c:2087
smartlist_clear
void smartlist_clear(smartlist_t *sl)
Definition: smartlist_core.c:50
padding_counts_t::read_cell_count
uint64_t read_cell_count
Definition: rephist.c:143
total_descriptor_downloads
static unsigned long total_descriptor_downloads
Definition: rephist.c:1643
circ_buffer_stats_t
Definition: rephist.c:1393
rep_hist_hsdir_stored_maybe_new_v3_onion
void rep_hist_hsdir_stored_maybe_new_v3_onion(const uint8_t *blinded_key)
Definition: rephist.c:1990
rep_hist_format_exit_stats
char * rep_hist_format_exit_stats(time_t now)
Definition: rephist.c:1171
rep_hist_note_circuit_handshake_assigned
void rep_hist_note_circuit_handshake_assigned(uint16_t type)
Definition: rephist.c:1806
connstats.h
Header for feature/stats/connstats.c.
or_history_t::changed
time_t changed
Definition: rephist.c:114
free_or_history
static void free_or_history(void *_hist)
Definition: rephist.c:348
networkstatus_t::valid_after
time_t valid_after
Definition: networkstatus_st.h:33
rep_hist_reset_desc_stats
static void rep_hist_reset_desc_stats(time_t now)
Definition: rephist.c:1664
tor_addr_is_null
int tor_addr_is_null(const tor_addr_t *addr)
Definition: address.c:780
exit_bytes_written
static uint64_t * exit_bytes_written
Definition: rephist.c:1120
tor_asprintf
int tor_asprintf(char **strp, const char *fmt,...)
Definition: printf.c:75
crypto_rand_double
double crypto_rand_double(void)
Definition: crypto_rand_numeric.c:126
SMARTLIST_FOREACH_BEGIN
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
Definition: smartlist_foreach.h:78
node_get_nickname
const char * node_get_nickname(const node_t *node)
Definition: nodelist.c:1450
rep_hist_seen_new_rp_cell
void rep_hist_seen_new_rp_cell(bool is_v2)
Definition: rephist.c:2016
or_options_t::TestingTorNetwork
int TestingTorNetwork
Definition: or_options_st.h:814
padding_published
static padding_counts_t padding_published
Definition: rephist.c:174
or_circuit_t::processed_cells
uint32_t processed_cells
Definition: or_circuit_st.h:74
start_of_buffer_stats_interval
static time_t start_of_buffer_stats_interval
Definition: rephist.c:1382
HEX_DIGEST_LEN
#define HEX_DIGEST_LEN
Definition: crypto_digest.h:35
padding_counts_t::enabled_read_pad_cell_count
uint64_t enabled_read_pad_cell_count
Definition: rephist.c:155
rep_hist_note_exit_bytes
void rep_hist_note_exit_bytes(uint16_t port, size_t num_written, size_t num_read)
Definition: rephist.c:1356
hs_v3_stats
static hs_v3_stats_t * hs_v3_stats
Definition: rephist.c:1928
or_history_t::since
time_t since
Definition: rephist.c:112
rep_hist_desc_stats_write
time_t rep_hist_desc_stats_write(time_t now)
Definition: rephist.c:1740
overload_happened_recently
static bool overload_happened_recently(time_t overload_time, int n_hours)
Definition: rephist.c:212
connection_or_digest_is_known_relay
int connection_or_digest_is_known_relay(const char *id_digest)
Definition: connection_or.c:814
order.h
Header for order.c.
format_local_iso_time
void format_local_iso_time(char *buf, time_t t)
Definition: time_fmt.c:285
or_circuit_t
Definition: or_circuit_st.h:21
correct_time
static time_t correct_time(time_t t, time_t now, time_t stored_at, time_t started_measuring)
Definition: rephist.c:894
config.h
Header file for config.c.
or_history_t::start_of_run
time_t start_of_run
Definition: rephist.c:129
PADDING_TYPE_DROP
@ PADDING_TYPE_DROP
Definition: rephist.h:123
smartlist_add_asprintf
void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern,...)
Definition: smartlist.c:36
rep_hist_hs_stats_write
time_t rep_hist_hs_stats_write(time_t now, bool is_v3)
Definition: rephist.c:2138
overload_stats
static overload_stats_t overload_stats
Definition: rephist.c:208
tor_log
void tor_log(int severity, log_domain_mask_t domain, const char *format,...)
Definition: log.c:590
rephist_total_alloc
uint64_t rephist_total_alloc
Definition: rephist.c:92
or_circuit_t::total_cell_waiting_time
uint64_t total_cell_waiting_time
Definition: or_circuit_st.h:79
start_of_served_descs_stats_interval
static time_t start_of_served_descs_stats_interval
Definition: rephist.c:1646
rep_hist_get_padding_count_lines
char * rep_hist_get_padding_count_lines(void)
Definition: rephist.c:2309
or_options_t
Definition: or_options_st.h:64
STATIC
#define STATIC
Definition: testsupport.h:32
networkstatus_t::fresh_until
time_t fresh_until
Definition: networkstatus_st.h:34
start_of_hs_v2_stats_interval
static time_t start_of_hs_v2_stats_interval
Definition: rephist.c:1851
rep_hist_exit_stats_write
time_t rep_hist_exit_stats_write(time_t now)
Definition: rephist.c:1326
MIN_CELL_COUNTS_TO_PUBLISH
#define MIN_CELL_COUNTS_TO_PUBLISH
Definition: rephist.c:2275
or_options_t::BandwidthBurst
uint64_t BandwidthBurst
Definition: or_options_st.h:418
rep_hist_reset_hs_v2_stats
static void rep_hist_reset_hs_v2_stats(time_t now)
Definition: rephist.c:1883
networkstatus_st.h
Networkstatus consensus/vote structure.
start_writing_to_stdio_file
FILE * start_writing_to_stdio_file(const char *fname, int open_flags, int mode, open_file_t **data_out)
Definition: files.c:398
smartlist_sort
void smartlist_sort(smartlist_t *sl, int(*compare)(const void **a, const void **b))
Definition: smartlist.c:334
TO_OR_CIRCUIT
or_circuit_t * TO_OR_CIRCUIT(circuit_t *x)
Definition: circuitlist.c:154
networkstatus_t
Definition: networkstatus_st.h:26
padding_counts_t::write_cell_count
uint64_t write_cell_count
Definition: rephist.c:145
start_of_exit_stats_interval
static time_t start_of_exit_stats_interval
Definition: rephist.c:1125
PADDING_TYPE_CELL
@ PADDING_TYPE_CELL
Definition: rephist.h:125
smartlist_t
Definition: smartlist_core.h:26
smartlist_join_strings
char * smartlist_join_strings(smartlist_t *sl, const char *join, int terminate, size_t *len_out)
Definition: smartlist.c:279
hs_get_start_time_of_next_time_period
time_t hs_get_start_time_of_next_time_period(time_t now)
Definition: hs_common.c:324
padding_counts_t::enabled_read_cell_count
uint64_t enabled_read_cell_count
Definition: rephist.c:151
rephist.h
Header file for rephist.c.
or_history_t::last_reached_port
uint16_t last_reached_port
Definition: rephist.c:121
connection_or.h
Header file for connection_or.c.
authdir_mode
int authdir_mode(const or_options_t *options)
Definition: authmode.c:25
tor_addr_copy
void tor_addr_copy(tor_addr_t *dest, const tor_addr_t *src)
Definition: address.c:933
history_map
static digestmap_t * history_map
Definition: rephist.c:184
rep_hist_log_circuit_handshake_stats
void rep_hist_log_circuit_handshake_stats(time_t now)
Definition: rephist.c:1834
or.h
Master header file for Tor-specific functionality.
rep_hist_record_mtbf_data
int rep_hist_record_mtbf_data(time_t now, int missing_means_down)
Definition: rephist.c:751