Tor  0.4.7.0-alpha-dev
rephist.c
Go to the documentation of this file.
1 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
2  * Copyright (c) 2007-2021, The Tor Project, Inc. */
3 /* See LICENSE for licensing information */
4 
5 /**
6  * \file rephist.c
7  * \brief Basic history and performance-tracking functionality.
8  *
9  * Basic history and performance-tracking functionality to remember
10  * which servers have worked in the past, how much bandwidth we've
11  * been using, which ports we tend to want, and so on; further,
12  * exit port statistics, cell statistics, and connection statistics.
13  *
14  * The history and information tracked in this module could sensibly be
15  * divided into several categories:
16  *
17  * <ul><li>Statistics used by authorities to remember the uptime and
18  * stability information about various relays, including "uptime",
19  * "weighted fractional uptime" and "mean time between failures".
20  *
21  * <li>Predicted ports, used by clients to remember how long it's been
22  * since they opened an exit connection to each given target
23  * port. Clients use this information in order to try to keep circuits
24  * open to exit nodes that can connect to the ports that they care
25  * about. (The predicted ports mechanism also handles predicted circuit
26  * usage that _isn't_ port-specific, such as resolves, internal circuits,
27  * and so on.)
28  *
29  * <li>Public key operation counters, for tracking how many times we've
30  * done each public key operation. (This is unmaintained and we should
31  * remove it.)
32  *
33  * <li>Exit statistics by port, used by exits to keep track of the
34  * number of streams and bytes they've served at each exit port, so they
35  * can generate their exit-kibibytes-{read,written} and
36  * exit-streams-opened statistics.
37  *
38  * <li>Circuit stats, used by relays instances to tract circuit
39  * queue fullness and delay over time, and generate cell-processed-cells,
40  * cell-queued-cells, cell-time-in-queue, and cell-circuits-per-decile
41  * statistics.
42  *
43  * <li>Descriptor serving statistics, used by directory caches to track
44  * how many descriptors they've served.
45  *
46  * <li>Onion handshake statistics, used by relays to count how many
47  * TAP and ntor handshakes they've handled.
48  *
49  * <li>Hidden service statistics, used by relays to count rendezvous
50  * traffic and HSDir-stored descriptors.
51  *
52  * <li>Link protocol statistics, used by relays to count how many times
53  * each link protocol has been used.
54  *
55  * </ul>
56  *
57  * The entry points for this module are scattered throughout the
58  * codebase. Sending data, receiving data, connecting to a relay,
59  * losing a connection to a relay, and so on can all trigger a change in
60  * our current stats. Relays also invoke this module in order to
61  * extract their statistics when building routerinfo and extrainfo
62  * objects in router.c.
63  *
64  * TODO: This module should be broken up.
65  *
66  * (The "rephist" name originally stood for "reputation and history". )
67  **/
68 
69 #define REPHIST_PRIVATE
70 #include "core/or/or.h"
71 #include "app/config/config.h"
72 #include "core/or/circuitlist.h"
73 #include "core/or/connection_or.h"
79 #include "feature/stats/rephist.h"
80 #include "lib/container/order.h"
82 #include "lib/math/laplace.h"
83 
85 #include "core/or/or_circuit_st.h"
86 
87 #include <event2/dns.h>
88 
89 #ifdef HAVE_FCNTL_H
90 #include <fcntl.h>
91 #endif
92 
93 /** Total number of bytes currently allocated in fields used by rephist.c. */
95 /** Number of or_history_t objects currently allocated. */
96 uint32_t rephist_total_num=0;
97 
98 /** If the total weighted run count of all runs for a router ever falls
99  * below this amount, the router can be treated as having 0 MTBF. */
100 #define STABILITY_EPSILON 0.0001
101 /** Value by which to discount all old intervals for MTBF purposes. This
102  * is compounded every STABILITY_INTERVAL. */
103 #define STABILITY_ALPHA 0.95
104 /** Interval at which to discount all old intervals for MTBF purposes. */
105 #define STABILITY_INTERVAL (12*60*60)
106 /* (This combination of ALPHA, INTERVAL, and EPSILON makes it so that an
107  * interval that just ended counts twice as much as one that ended a week ago,
108  * 20X as much as one that ended a month ago, and routers that have had no
109  * uptime data for about half a year will get forgotten.) */
110 
111 /** History of an OR. */
112 typedef struct or_history_t {
113  /** When did we start tracking this OR? */
114  time_t since;
115  /** When did we most recently note a change to this OR? */
116  time_t changed;
117 
118  /** The address at which we most recently connected to this OR
119  * successfully. */
121 
122  /** The port at which we most recently connected to this OR successfully */
124 
125  /* === For MTBF tracking: */
126  /** Weighted sum total of all times that this router has been online.
127  */
128  unsigned long weighted_run_length;
129  /** If the router is now online (according to stability-checking rules),
130  * when did it come online? */
131  time_t start_of_run;
132  /** Sum of weights for runs in weighted_run_length. */
134  /* === For fractional uptime tracking: */
135  time_t start_of_downtime;
136  unsigned long weighted_uptime;
137  unsigned long total_weighted_time;
138 } or_history_t;
139 
140 /**
141  * This structure holds accounting needed to calculate the padding overhead.
142  */
143 typedef struct padding_counts_t {
144  /** Total number of cells we have received, including padding */
145  uint64_t read_cell_count;
146  /** Total number of cells we have sent, including padding */
148  /** Total number of CELL_PADDING cells we have received */
150  /** Total number of CELL_PADDING cells we have sent */
152  /** Total number of read cells on padding-enabled conns */
154  /** Total number of sent cells on padding-enabled conns */
156  /** Total number of read CELL_PADDING cells on padding-enabled cons */
158  /** Total number of sent CELL_PADDING cells on padding-enabled cons */
160  /** Total number of RELAY_DROP cells we have received */
162  /** Total number of RELAY_DROP cells we have sent */
164  /** The maximum number of padding timers we've seen in 24 hours */
166  /** When did we first copy padding_current into padding_published? */
167  char first_published_at[ISO_TIME_LEN+1];
169 
170 /** Holds the current values of our padding statistics.
171  * It is not published until it is transferred to padding_published. */
173 
174 /** Remains fixed for a 24 hour period, and then is replaced
175  * by a redacted copy of padding_current */
177 
178 /** When did we last multiply all routers' weighted_run_length and
179  * total_run_weights by STABILITY_ALPHA? */
180 static time_t stability_last_downrated = 0;
181 
182 /** */
183 static time_t started_tracking_stability = 0;
184 
185 /** Map from hex OR identity digest to or_history_t. */
186 static digestmap_t *history_map = NULL;
187 
188 /** Represents a state of overload stats.
189  *
190  * All the timestamps in this structure have already been rounded down to the
191  * nearest hour. */
192 typedef struct {
193  /* When did we last experience a general overload? */
194  time_t overload_general_time;
195 
196  /* When did we last experience a bandwidth-related overload? */
197  time_t overload_ratelimits_time;
198  /* How many times have we gone off the our read limits? */
199  uint64_t overload_read_count;
200  /* How many times have we gone off the our write limits? */
201  uint64_t overload_write_count;
202 
203  /* When did we last experience a file descriptor exhaustion? */
204  time_t overload_fd_exhausted_time;
205  /* How many times have we experienced a file descriptor exhaustion? */
206  uint64_t overload_fd_exhausted;
208 
209 /** Current state of overload stats */
211 
212 /** Counters to count the number of times we've reached an overload for the
213  * global connection read/write limit. Reported on the MetricsPort. */
214 static uint64_t stats_n_read_limit_reached = 0;
215 static uint64_t stats_n_write_limit_reached = 0;
216 
217 /** Total number of times we've reached TCP port exhaustion. */
218 static uint64_t stats_n_tcp_exhaustion = 0;
219 
220 /***** DNS statistics *****/
221 
222 /** Represents the statistics of DNS queries seen if it is an Exit. */
223 typedef struct {
224  /* Total number of DNS errors found in RFC 1035 (from 0 to 5 code). */
225  uint64_t stats_n_error_none; /* 0 */
226  uint64_t stats_n_error_format; /* 1 */
227  uint64_t stats_n_error_serverfailed; /* 2 */
228  uint64_t stats_n_error_notexist; /* 3 */
229  uint64_t stats_n_error_notimpl; /* 4 */
230  uint64_t stats_n_error_refused; /* 5 */
231 
232  /* Total number of DNS errors specific to libevent. */
233  uint64_t stats_n_error_truncated; /* 65 */
234  uint64_t stats_n_error_unknown; /* 66 */
235  uint64_t stats_n_error_timeout; /* 67 */
236  uint64_t stats_n_error_shutdown; /* 68 */
237  uint64_t stats_n_error_cancel; /* 69 */
238  uint64_t stats_n_error_nodata; /* 70 */
239 
240  /* Total number of DNS request seen at an Exit. They might not all end
241  * successfully or might even be lost by tor. This counter is incremented
242  * right before the DNS request is initiated. */
243  uint64_t stats_n_request;
244 } dns_stats_t;
245 
246 /** DNS statistics store for each DNS record type for which tor supports only
247  * three at the moment: A, PTR and AAAA. */
249 static dns_stats_t dns_PTR_stats;
250 static dns_stats_t dns_AAAA_stats;
251 
252 /** From a libevent record type, return a pointer to the corresponding DNS
253  * statistics store. NULL is returned if the type is unhandled. */
254 static inline dns_stats_t *
255 get_dns_stats_by_type(const int type)
256 {
257  switch (type) {
258  case DNS_IPv4_A:
259  return &dns_A_stats;
260  case DNS_PTR:
261  return &dns_PTR_stats;
262  case DNS_IPv6_AAAA:
263  return &dns_AAAA_stats;
264  default:
265  return NULL;
266  }
267 }
268 
269 /** Return the DNS error count for the given libevent DNS type and error code.
270  * The possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. */
271 uint64_t
272 rep_hist_get_n_dns_error(int type, uint8_t error)
273 {
274  dns_stats_t *dns_stats = get_dns_stats_by_type(type);
275  if (BUG(!dns_stats)) {
276  return 0;
277  }
278 
279  switch (error) {
280  case DNS_ERR_NONE:
281  return dns_stats->stats_n_error_none;
282  case DNS_ERR_FORMAT:
283  return dns_stats->stats_n_error_format;
284  case DNS_ERR_SERVERFAILED:
285  return dns_stats->stats_n_error_serverfailed;
286  case DNS_ERR_NOTEXIST:
287  return dns_stats->stats_n_error_notexist;
288  case DNS_ERR_NOTIMPL:
289  return dns_stats->stats_n_error_notimpl;
290  case DNS_ERR_REFUSED:
291  return dns_stats->stats_n_error_refused;
292  case DNS_ERR_TRUNCATED:
293  return dns_stats->stats_n_error_truncated;
294  case DNS_ERR_UNKNOWN:
295  return dns_stats->stats_n_error_unknown;
296  case DNS_ERR_TIMEOUT:
297  return dns_stats->stats_n_error_timeout;
298  case DNS_ERR_SHUTDOWN:
299  return dns_stats->stats_n_error_shutdown;
300  case DNS_ERR_CANCEL:
301  return dns_stats->stats_n_error_cancel;
302  case DNS_ERR_NODATA:
303  return dns_stats->stats_n_error_nodata;
304  default:
305  /* Unhandled code sent back by libevent. */
306  return 0;
307  }
308 }
309 
310 /** Return the total number of DNS request seen for the given libevent DNS
311  * record type. Possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. */
312 uint64_t
314 {
315  dns_stats_t *dns_stats = get_dns_stats_by_type(type);
316  if (BUG(!dns_stats)) {
317  return 0;
318  }
319  return dns_stats->stats_n_request;
320 }
321 
322 /** Note a DNS error for the given given libevent DNS record type and error
323  * code. Possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. */
324 void
325 rep_hist_note_dns_error(int type, uint8_t error)
326 {
327  dns_stats_t *dns_stats = get_dns_stats_by_type(type);
328  /* Unsupported DNS query type. */
329  if (!dns_stats) {
330  return;
331  }
332 
333  switch (error) {
334  case DNS_ERR_NONE:
335  dns_stats->stats_n_error_none++;
336  break;
337  case DNS_ERR_FORMAT:
338  dns_stats->stats_n_error_format++;
339  break;
340  case DNS_ERR_SERVERFAILED:
341  dns_stats->stats_n_error_serverfailed++;
342  break;
343  case DNS_ERR_NOTEXIST:
344  dns_stats->stats_n_error_notexist++;
345  break;
346  case DNS_ERR_NOTIMPL:
347  dns_stats->stats_n_error_notimpl++;
348  break;
349  case DNS_ERR_REFUSED:
350  dns_stats->stats_n_error_refused++;
351  break;
352  case DNS_ERR_TRUNCATED:
353  dns_stats->stats_n_error_truncated++;
354  break;
355  case DNS_ERR_UNKNOWN:
356  dns_stats->stats_n_error_unknown++;
357  break;
358  case DNS_ERR_TIMEOUT:
359  dns_stats->stats_n_error_timeout++;
360  break;
361  case DNS_ERR_SHUTDOWN:
362  dns_stats->stats_n_error_shutdown++;
363  break;
364  case DNS_ERR_CANCEL:
365  dns_stats->stats_n_error_cancel++;
366  break;
367  case DNS_ERR_NODATA:
368  dns_stats->stats_n_error_nodata++;
369  break;
370  default:
371  /* Unhandled code sent back by libevent. */
372  break;
373  }
374 }
375 
376 /** Note a DNS request for the given given libevent DNS record type. */
377 void
379 {
380  dns_stats_t *dns_stats = get_dns_stats_by_type(type);
381  if (BUG(!dns_stats)) {
382  return;
383  }
384  dns_stats->stats_n_request++;
385 }
386 
387 /***** END of DNS statistics *****/
388 
389 /** Return true if this overload happened within the last `n_hours`. */
390 static bool
391 overload_happened_recently(time_t overload_time, int n_hours)
392 {
393  /* An overload is relevant if it happened in the last 72 hours */
394  if (overload_time > approx_time() - 3600 * n_hours) {
395  return true;
396  }
397  return false;
398 }
399 
400 /* The current version of the overload stats version */
401 #define OVERLOAD_STATS_VERSION 1
402 
403 /** Return the stats_n_read_limit_reached counter. */
404 uint64_t
406 {
408 }
409 
410 /** Return the stats_n_write_limit_reached counter. */
411 uint64_t
413 {
414  return stats_n_write_limit_reached;
415 }
416 
417 /** Returns an allocated string for server descriptor for publising information
418  * on whether we are overloaded or not. */
419 char *
421 {
422  char *result = NULL;
423  char tbuf[ISO_TIME_LEN+1];
424 
425  /* Encode the general overload */
426  if (overload_happened_recently(overload_stats.overload_general_time, 72)) {
427  format_iso_time(tbuf, overload_stats.overload_general_time);
428  tor_asprintf(&result, "overload-general %d %s\n",
429  OVERLOAD_STATS_VERSION, tbuf);
430  }
431 
432  return result;
433 }
434 
435 /** Returns an allocated string for extra-info documents for publishing
436  * overload statistics. */
437 char *
439 {
440  char *result = NULL;
441  smartlist_t *chunks = smartlist_new();
442  char tbuf[ISO_TIME_LEN+1];
443 
444  /* Add bandwidth-related overloads */
445  if (overload_happened_recently(overload_stats.overload_ratelimits_time,24)) {
446  const or_options_t *options = get_options();
447  format_iso_time(tbuf, overload_stats.overload_ratelimits_time);
448  smartlist_add_asprintf(chunks,
449  "overload-ratelimits %d %s %" PRIu64 " %" PRIu64
450  " %" PRIu64 " %" PRIu64 "\n",
451  OVERLOAD_STATS_VERSION, tbuf,
452  options->BandwidthRate, options->BandwidthBurst,
453  overload_stats.overload_read_count,
454  overload_stats.overload_write_count);
455  }
456 
457  /* Finally file descriptor overloads */
459  overload_stats.overload_fd_exhausted_time, 72)) {
460  format_iso_time(tbuf, overload_stats.overload_fd_exhausted_time);
461  smartlist_add_asprintf(chunks, "overload-fd-exhausted %d %s\n",
462  OVERLOAD_STATS_VERSION, tbuf);
463  }
464 
465  /* Bail early if we had nothing to write */
466  if (smartlist_len(chunks) == 0) {
467  goto done;
468  }
469 
470  result = smartlist_join_strings(chunks, "", 0, NULL);
471 
472  done:
473  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
474  smartlist_free(chunks);
475  return result;
476 }
477 
478 /** Round down the time in `a` to the beginning of the current hour */
479 #define SET_TO_START_OF_HOUR(a) STMT_BEGIN \
480  (a) = approx_time() - (approx_time() % 3600); \
481 STMT_END
482 
483 /** Note down an overload event of type `overload`. */
484 void
486 {
487  static time_t last_read_counted = 0;
488  static time_t last_write_counted = 0;
489 
490  switch (overload) {
491  case OVERLOAD_GENERAL:
492  SET_TO_START_OF_HOUR(overload_stats.overload_general_time);
493  break;
494  case OVERLOAD_READ: {
496  SET_TO_START_OF_HOUR(overload_stats.overload_ratelimits_time);
497  if (approx_time() >= last_read_counted + 60) { /* Count once a minute */
498  overload_stats.overload_read_count++;
499  last_read_counted = approx_time();
500  }
501  break;
502  }
503  case OVERLOAD_WRITE: {
504  stats_n_write_limit_reached++;
505  SET_TO_START_OF_HOUR(overload_stats.overload_ratelimits_time);
506  if (approx_time() >= last_write_counted + 60) { /* Count once a minute */
507  overload_stats.overload_write_count++;
508  last_write_counted = approx_time();
509  }
510  break;
511  }
512  case OVERLOAD_FD_EXHAUSTED:
513  SET_TO_START_OF_HOUR(overload_stats.overload_fd_exhausted_time);
514  overload_stats.overload_fd_exhausted++;
515  break;
516  }
517 }
518 
519 /** Note down that we've reached a TCP port exhaustion. This triggers an
520  * overload general event. */
521 void
523 {
525  rep_hist_note_overload(OVERLOAD_GENERAL);
526 }
527 
528 /** Return the total number of TCP exhaustion times we've reached. */
529 uint64_t
531 {
532  return stats_n_tcp_exhaustion;
533 }
534 
535 /** Return the or_history_t for the OR with identity digest <b>id</b>,
536  * creating it if necessary. */
537 static or_history_t *
538 get_or_history(const char* id)
539 {
540  or_history_t *hist;
541 
542  if (tor_digest_is_zero(id))
543  return NULL;
544 
545  hist = digestmap_get(history_map, id);
546  if (!hist) {
547  hist = tor_malloc_zero(sizeof(or_history_t));
550  hist->since = hist->changed = time(NULL);
552  digestmap_set(history_map, id, hist);
553  }
554  return hist;
555 }
556 
557 /** Helper: free storage held by a single OR history entry. */
558 static void
559 free_or_history(void *_hist)
560 {
561  or_history_t *hist = _hist;
564  tor_free(hist);
565 }
566 
567 /** Initialize the static data structures for tracking history. */
568 void
570 {
571  history_map = digestmap_new();
572 }
573 
574 /** We have just decided that this router with identity digest <b>id</b> is
575  * reachable, meaning we will give it a "Running" flag for the next while. */
576 void
577 rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr,
578  const uint16_t at_port, time_t when)
579 {
580  or_history_t *hist = get_or_history(id);
581  int was_in_run = 1;
582  char tbuf[ISO_TIME_LEN+1];
583  int addr_changed, port_changed;
584 
585  tor_assert(hist);
586  tor_assert((!at_addr && !at_port) || (at_addr && at_port));
587 
588  addr_changed = at_addr && !tor_addr_is_null(&hist->last_reached_addr) &&
589  tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0;
590  port_changed = at_port && hist->last_reached_port &&
591  at_port != hist->last_reached_port;
592 
593  if (!started_tracking_stability)
594  started_tracking_stability = time(NULL);
595  if (!hist->start_of_run) {
596  hist->start_of_run = when;
597  was_in_run = 0;
598  }
599  if (hist->start_of_downtime) {
600  long down_length;
601 
602  format_local_iso_time(tbuf, hist->start_of_downtime);
603  log_info(LD_HIST, "Router %s is now Running; it had been down since %s.",
604  hex_str(id, DIGEST_LEN), tbuf);
605  if (was_in_run)
606  log_info(LD_HIST, " (Paradoxically, it was already Running too.)");
607 
608  down_length = when - hist->start_of_downtime;
609  hist->total_weighted_time += down_length;
610  hist->start_of_downtime = 0;
611  } else if (addr_changed || port_changed) {
612  /* If we're reachable, but the address changed, treat this as some
613  * downtime. */
614  int penalty = get_options()->TestingTorNetwork ? 240 : 3600;
615  networkstatus_t *ns;
616 
617  if ((ns = networkstatus_get_latest_consensus())) {
618  int fresh_interval = (int)(ns->fresh_until - ns->valid_after);
619  int live_interval = (int)(ns->valid_until - ns->valid_after);
620  /* on average, a descriptor addr change takes .5 intervals to make it
621  * into a consensus, and half a liveness period to make it to
622  * clients. */
623  penalty = (int)(fresh_interval + live_interval) / 2;
624  }
625  format_local_iso_time(tbuf, hist->start_of_run);
626  log_info(LD_HIST,"Router %s still seems Running, but its address appears "
627  "to have changed since the last time it was reachable. I'm "
628  "going to treat it as having been down for %d seconds",
629  hex_str(id, DIGEST_LEN), penalty);
630  rep_hist_note_router_unreachable(id, when-penalty);
631  rep_hist_note_router_reachable(id, NULL, 0, when);
632  } else {
633  format_local_iso_time(tbuf, hist->start_of_run);
634  if (was_in_run)
635  log_debug(LD_HIST, "Router %s is still Running; it has been Running "
636  "since %s", hex_str(id, DIGEST_LEN), tbuf);
637  else
638  log_info(LD_HIST,"Router %s is now Running; it was previously untracked",
639  hex_str(id, DIGEST_LEN));
640  }
641  if (at_addr)
642  tor_addr_copy(&hist->last_reached_addr, at_addr);
643  if (at_port)
644  hist->last_reached_port = at_port;
645 }
646 
647 /** We have just decided that this router is unreachable, meaning
648  * we are taking away its "Running" flag. */
649 void
650 rep_hist_note_router_unreachable(const char *id, time_t when)
651 {
652  or_history_t *hist = get_or_history(id);
653  char tbuf[ISO_TIME_LEN+1];
654  int was_running = 0;
655  if (!started_tracking_stability)
656  started_tracking_stability = time(NULL);
657 
658  tor_assert(hist);
659  if (hist->start_of_run) {
660  /*XXXX We could treat failed connections differently from failed
661  * connect attempts. */
662  long run_length = when - hist->start_of_run;
663  format_local_iso_time(tbuf, hist->start_of_run);
664 
665  hist->total_run_weights += 1.0;
666  hist->start_of_run = 0;
667  if (run_length < 0) {
668  unsigned long penalty = -run_length;
669 #define SUBTRACT_CLAMPED(var, penalty) \
670  do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0)
671 
672  SUBTRACT_CLAMPED(hist->weighted_run_length, penalty);
673  SUBTRACT_CLAMPED(hist->weighted_uptime, penalty);
674  } else {
675  hist->weighted_run_length += run_length;
676  hist->weighted_uptime += run_length;
677  hist->total_weighted_time += run_length;
678  }
679  was_running = 1;
680  log_info(LD_HIST, "Router %s is now non-Running: it had previously been "
681  "Running since %s. Its total weighted uptime is %lu/%lu.",
682  hex_str(id, DIGEST_LEN), tbuf, hist->weighted_uptime,
683  hist->total_weighted_time);
684  }
685  if (!hist->start_of_downtime) {
686  hist->start_of_downtime = when;
687 
688  if (!was_running)
689  log_info(LD_HIST, "Router %s is now non-Running; it was previously "
690  "untracked.", hex_str(id, DIGEST_LEN));
691  } else {
692  if (!was_running) {
693  format_local_iso_time(tbuf, hist->start_of_downtime);
694 
695  log_info(LD_HIST, "Router %s is still non-Running; it has been "
696  "non-Running since %s.", hex_str(id, DIGEST_LEN), tbuf);
697  }
698  }
699 }
700 
701 /** Mark a router with ID <b>id</b> as non-Running, and retroactively declare
702  * that it has never been running: give it no stability and no WFU. */
703 void
704 rep_hist_make_router_pessimal(const char *id, time_t when)
705 {
706  or_history_t *hist = get_or_history(id);
707  tor_assert(hist);
708 
710 
711  hist->weighted_run_length = 0;
712  hist->weighted_uptime = 0;
713 }
714 
715 /** Helper: Discount all old MTBF data, if it is time to do so. Return
716  * the time at which we should next discount MTBF data. */
717 time_t
719 {
720  digestmap_iter_t *orhist_it;
721  const char *digest1;
722  or_history_t *hist;
723  void *hist_p;
724  double alpha = 1.0;
725 
726  if (!history_map)
727  history_map = digestmap_new();
732 
733  /* Okay, we should downrate the data. By how much? */
736  alpha *= STABILITY_ALPHA;
737  }
738 
739  log_info(LD_HIST, "Discounting all old stability info by a factor of %f",
740  alpha);
741 
742  /* Multiply every w_r_l, t_r_w pair by alpha. */
743  for (orhist_it = digestmap_iter_init(history_map);
744  !digestmap_iter_done(orhist_it);
745  orhist_it = digestmap_iter_next(history_map,orhist_it)) {
746  digestmap_iter_get(orhist_it, &digest1, &hist_p);
747  hist = hist_p;
748 
749  hist->weighted_run_length =
750  (unsigned long)(hist->weighted_run_length * alpha);
751  hist->total_run_weights *= alpha;
752 
753  hist->weighted_uptime = (unsigned long)(hist->weighted_uptime * alpha);
754  hist->total_weighted_time = (unsigned long)
755  (hist->total_weighted_time * alpha);
756  }
757 
759 }
760 
761 /** Helper: Return the weighted MTBF of the router with history <b>hist</b>. */
762 static double
763 get_stability(or_history_t *hist, time_t when)
764 {
765  long total = hist->weighted_run_length;
766  double total_weights = hist->total_run_weights;
767 
768  if (hist->start_of_run) {
769  /* We're currently in a run. Let total and total_weights hold the values
770  * they would hold if the current run were to end now. */
771  total += (when-hist->start_of_run);
772  total_weights += 1.0;
773  }
774  if (total_weights < STABILITY_EPSILON) {
775  /* Round down to zero, and avoid divide-by-zero. */
776  return 0.0;
777  }
778 
779  return total / total_weights;
780 }
781 
782 /** Return the total amount of time we've been observing, with each run of
783  * time downrated by the appropriate factor. */
784 static long
786 {
787  long total = hist->total_weighted_time;
788  if (hist->start_of_run) {
789  total += (when - hist->start_of_run);
790  } else if (hist->start_of_downtime) {
791  total += (when - hist->start_of_downtime);
792  }
793  return total;
794 }
795 
796 /** Helper: Return the weighted percent-of-time-online of the router with
797  * history <b>hist</b>. */
798 static double
800 {
801  long total = hist->total_weighted_time;
802  long up = hist->weighted_uptime;
803 
804  if (hist->start_of_run) {
805  long run_length = (when - hist->start_of_run);
806  up += run_length;
807  total += run_length;
808  } else if (hist->start_of_downtime) {
809  total += (when - hist->start_of_downtime);
810  }
811 
812  if (!total) {
813  /* Avoid calling anybody's uptime infinity (which should be impossible if
814  * the code is working), or NaN (which can happen for any router we haven't
815  * observed up or down yet). */
816  return 0.0;
817  }
818 
819  return ((double) up) / total;
820 }
821 
822 /** Return how long the router whose identity digest is <b>id</b> has
823  * been reachable. Return 0 if the router is unknown or currently deemed
824  * unreachable. */
825 long
826 rep_hist_get_uptime(const char *id, time_t when)
827 {
828  or_history_t *hist = get_or_history(id);
829  if (!hist)
830  return 0;
831  if (!hist->start_of_run || when < hist->start_of_run)
832  return 0;
833  return when - hist->start_of_run;
834 }
835 
836 /** Return an estimated MTBF for the router whose identity digest is
837  * <b>id</b>. Return 0 if the router is unknown. */
838 double
839 rep_hist_get_stability(const char *id, time_t when)
840 {
841  or_history_t *hist = get_or_history(id);
842  if (!hist)
843  return 0.0;
844 
845  return get_stability(hist, when);
846 }
847 
848 /** Return an estimated percent-of-time-online for the router whose identity
849  * digest is <b>id</b>. Return 0 if the router is unknown. */
850 double
851 rep_hist_get_weighted_fractional_uptime(const char *id, time_t when)
852 {
853  or_history_t *hist = get_or_history(id);
854  if (!hist)
855  return 0.0;
856 
857  return get_weighted_fractional_uptime(hist, when);
858 }
859 
860 /** Return a number representing how long we've known about the router whose
861  * digest is <b>id</b>. Return 0 if the router is unknown.
862  *
863  * Be careful: this measure increases monotonically as we know the router for
864  * longer and longer, but it doesn't increase linearly.
865  */
866 long
867 rep_hist_get_weighted_time_known(const char *id, time_t when)
868 {
869  or_history_t *hist = get_or_history(id);
870  if (!hist)
871  return 0;
872 
873  return get_total_weighted_time(hist, when);
874 }
875 
876 /** Return true if we've been measuring MTBFs for long enough to
877  * pronounce on Stability. */
878 int
880 {
881  /* XXXX++ This doesn't do so well when we change our opinion
882  * as to whether we're tracking router stability. */
883  return started_tracking_stability < time(NULL) - 4*60*60;
884 }
885 
886 /** Log all the reliability data we have remembered, with the chosen
887  * severity.
888  */
889 void
890 rep_hist_dump_stats(time_t now, int severity)
891 {
892  digestmap_iter_t *orhist_it;
893  const char *name1, *digest1;
894  char hexdigest1[HEX_DIGEST_LEN+1];
895  or_history_t *or_history;
896  void *or_history_p;
897  const node_t *node;
898 
899  rep_history_clean(now - get_options()->RephistTrackTime);
900 
901  tor_log(severity, LD_HIST, "--------------- Dumping history information:");
902 
903  for (orhist_it = digestmap_iter_init(history_map);
904  !digestmap_iter_done(orhist_it);
905  orhist_it = digestmap_iter_next(history_map,orhist_it)) {
906  double s;
907  long stability;
908  digestmap_iter_get(orhist_it, &digest1, &or_history_p);
909  or_history = (or_history_t*) or_history_p;
910 
911  if ((node = node_get_by_id(digest1)) && node_get_nickname(node))
912  name1 = node_get_nickname(node);
913  else
914  name1 = "(unknown)";
915  base16_encode(hexdigest1, sizeof(hexdigest1), digest1, DIGEST_LEN);
916  s = get_stability(or_history, now);
917  stability = (long)s;
918  tor_log(severity, LD_HIST,
919  "OR %s [%s]: wmtbf %lu:%02lu:%02lu",
920  name1, hexdigest1,
921  stability/3600, (stability/60)%60, stability%60);
922  }
923 }
924 
925 /** Remove history info for routers/links that haven't changed since
926  * <b>before</b>.
927  */
928 void
929 rep_history_clean(time_t before)
930 {
931  int authority = authdir_mode(get_options());
932  or_history_t *or_history;
933  void *or_history_p;
934  digestmap_iter_t *orhist_it;
935  const char *d1;
936 
937  orhist_it = digestmap_iter_init(history_map);
938  while (!digestmap_iter_done(orhist_it)) {
939  int should_remove;
940  digestmap_iter_get(orhist_it, &d1, &or_history_p);
941  or_history = or_history_p;
942 
943  should_remove = authority ?
944  (or_history->total_run_weights < STABILITY_EPSILON &&
945  !or_history->start_of_run)
946  : (or_history->changed < before);
947  if (should_remove) {
948  orhist_it = digestmap_iter_next_rmv(history_map, orhist_it);
949  free_or_history(or_history);
950  continue;
951  }
952  orhist_it = digestmap_iter_next(history_map, orhist_it);
953  }
954 }
955 
956 /** Write MTBF data to disk. Return 0 on success, negative on failure.
957  *
958  * If <b>missing_means_down</b>, then if we're about to write an entry
959  * that is still considered up but isn't in our routerlist, consider it
960  * to be down. */
961 int
962 rep_hist_record_mtbf_data(time_t now, int missing_means_down)
963 {
964  char time_buf[ISO_TIME_LEN+1];
965 
966  digestmap_iter_t *orhist_it;
967  const char *digest;
968  void *or_history_p;
969  or_history_t *hist;
970  open_file_t *open_file = NULL;
971  FILE *f;
972 
973  {
974  char *filename = get_datadir_fname("router-stability");
975  f = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE|O_TEXT, 0600,
976  &open_file);
977  tor_free(filename);
978  if (!f)
979  return -1;
980  }
981 
982  /* File format is:
983  * FormatLine *KeywordLine Data
984  *
985  * FormatLine = "format 1" NL
986  * KeywordLine = Keyword SP Arguments NL
987  * Data = "data" NL *RouterMTBFLine "." NL
988  * RouterMTBFLine = Fingerprint SP WeightedRunLen SP
989  * TotalRunWeights [SP S=StartRunTime] NL
990  */
991 #define PUT(s) STMT_BEGIN if (fputs((s),f)<0) goto err; STMT_END
992 #define PRINTF(args) STMT_BEGIN if (fprintf args <0) goto err; STMT_END
993 
994  PUT("format 2\n");
995 
996  format_iso_time(time_buf, time(NULL));
997  PRINTF((f, "stored-at %s\n", time_buf));
998 
999  if (started_tracking_stability) {
1000  format_iso_time(time_buf, started_tracking_stability);
1001  PRINTF((f, "tracked-since %s\n", time_buf));
1002  }
1005  PRINTF((f, "last-downrated %s\n", time_buf));
1006  }
1007 
1008  PUT("data\n");
1009 
1010  /* XXX Nick: now bridge auths record this for all routers too.
1011  * Should we make them record it only for bridge routers? -RD
1012  * Not for 0.2.0. -NM */
1013  for (orhist_it = digestmap_iter_init(history_map);
1014  !digestmap_iter_done(orhist_it);
1015  orhist_it = digestmap_iter_next(history_map,orhist_it)) {
1016  char dbuf[HEX_DIGEST_LEN+1];
1017  const char *t = NULL;
1018  digestmap_iter_get(orhist_it, &digest, &or_history_p);
1019  hist = (or_history_t*) or_history_p;
1020 
1021  base16_encode(dbuf, sizeof(dbuf), digest, DIGEST_LEN);
1022 
1023  if (missing_means_down && hist->start_of_run &&
1025  /* We think this relay is running, but it's not listed in our
1026  * consensus. Somehow it fell out without telling us it went
1027  * down. Complain and also correct it. */
1028  log_info(LD_HIST,
1029  "Relay '%s' is listed as up in rephist, but it's not in "
1030  "our routerlist. Correcting.", dbuf);
1031  rep_hist_note_router_unreachable(digest, now);
1032  }
1033 
1034  PRINTF((f, "R %s\n", dbuf));
1035  if (hist->start_of_run > 0) {
1036  format_iso_time(time_buf, hist->start_of_run);
1037  t = time_buf;
1038  }
1039  PRINTF((f, "+MTBF %lu %.5f%s%s\n",
1041  t ? " S=" : "", t ? t : ""));
1042  t = NULL;
1043  if (hist->start_of_downtime > 0) {
1044  format_iso_time(time_buf, hist->start_of_downtime);
1045  t = time_buf;
1046  }
1047  PRINTF((f, "+WFU %lu %lu%s%s\n",
1048  hist->weighted_uptime, hist->total_weighted_time,
1049  t ? " S=" : "", t ? t : ""));
1050  }
1051 
1052  PUT(".\n");
1053 
1054 #undef PUT
1055 #undef PRINTF
1056 
1057  return finish_writing_to_file(open_file);
1058  err:
1059  abort_writing_to_file(open_file);
1060  return -1;
1061 }
1062 
1063 /** Helper: return the first j >= i such that !strcmpstart(sl[j], prefix) and
1064  * such that no line sl[k] with i <= k < j starts with "R ". Return -1 if no
1065  * such line exists. */
1066 static int
1067 find_next_with(smartlist_t *sl, int i, const char *prefix)
1068 {
1069  for ( ; i < smartlist_len(sl); ++i) {
1070  const char *line = smartlist_get(sl, i);
1071  if (!strcmpstart(line, prefix))
1072  return i;
1073  if (!strcmpstart(line, "R "))
1074  return -1;
1075  }
1076  return -1;
1077 }
1078 
1079 /** How many bad times has parse_possibly_bad_iso_time() parsed? */
1080 static int n_bogus_times = 0;
1081 /** Parse the ISO-formatted time in <b>s</b> into *<b>time_out</b>, but
1082  * round any pre-1970 date to Jan 1, 1970. */
1083 static int
1084 parse_possibly_bad_iso_time(const char *s, time_t *time_out)
1085 {
1086  int year;
1087  char b[5];
1088  strlcpy(b, s, sizeof(b));
1089  b[4] = '\0';
1090  year = (int)tor_parse_long(b, 10, 0, INT_MAX, NULL, NULL);
1091  if (year < 1970) {
1092  *time_out = 0;
1093  ++n_bogus_times;
1094  return 0;
1095  } else
1096  return parse_iso_time(s, time_out);
1097 }
1098 
1099 /** We've read a time <b>t</b> from a file stored at <b>stored_at</b>, which
1100  * says we started measuring at <b>started_measuring</b>. Return a new number
1101  * that's about as much before <b>now</b> as <b>t</b> was before
1102  * <b>stored_at</b>.
1103  */
1104 static inline time_t
1105 correct_time(time_t t, time_t now, time_t stored_at, time_t started_measuring)
1106 {
1107  if (t < started_measuring - 24*60*60*365)
1108  return 0;
1109  else if (t < started_measuring)
1110  return started_measuring;
1111  else if (t > stored_at)
1112  return 0;
1113  else {
1114  long run_length = stored_at - t;
1115  t = (time_t)(now - run_length);
1116  if (t < started_measuring)
1117  t = started_measuring;
1118  return t;
1119  }
1120 }
1121 
1122 /** Load MTBF data from disk. Returns 0 on success or recoverable error, -1
1123  * on failure. */
1124 int
1126 {
1127  /* XXXX won't handle being called while history is already populated. */
1128  smartlist_t *lines;
1129  const char *line = NULL;
1130  int r=0, i;
1131  time_t last_downrated = 0, stored_at = 0, tracked_since = 0;
1132  time_t latest_possible_start = now;
1133  long format = -1;
1134 
1135  {
1136  char *filename = get_datadir_fname("router-stability");
1137  char *d = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
1138  tor_free(filename);
1139  if (!d)
1140  return -1;
1141  lines = smartlist_new();
1142  smartlist_split_string(lines, d, "\n", SPLIT_SKIP_SPACE, 0);
1143  tor_free(d);
1144  }
1145 
1146  {
1147  const char *firstline;
1148  if (smartlist_len(lines)>4) {
1149  firstline = smartlist_get(lines, 0);
1150  if (!strcmpstart(firstline, "format "))
1151  format = tor_parse_long(firstline+strlen("format "),
1152  10, -1, LONG_MAX, NULL, NULL);
1153  }
1154  }
1155  if (format != 1 && format != 2) {
1156  log_warn(LD_HIST,
1157  "Unrecognized format in mtbf history file. Skipping.");
1158  goto err;
1159  }
1160  for (i = 1; i < smartlist_len(lines); ++i) {
1161  line = smartlist_get(lines, i);
1162  if (!strcmp(line, "data"))
1163  break;
1164  if (!strcmpstart(line, "last-downrated ")) {
1165  if (parse_iso_time(line+strlen("last-downrated "), &last_downrated)<0)
1166  log_warn(LD_HIST,"Couldn't parse downrate time in mtbf "
1167  "history file.");
1168  }
1169  if (!strcmpstart(line, "stored-at ")) {
1170  if (parse_iso_time(line+strlen("stored-at "), &stored_at)<0)
1171  log_warn(LD_HIST,"Couldn't parse stored time in mtbf "
1172  "history file.");
1173  }
1174  if (!strcmpstart(line, "tracked-since ")) {
1175  if (parse_iso_time(line+strlen("tracked-since "), &tracked_since)<0)
1176  log_warn(LD_HIST,"Couldn't parse started-tracking time in mtbf "
1177  "history file.");
1178  }
1179  }
1180  if (last_downrated > now)
1181  last_downrated = now;
1182  if (tracked_since > now)
1183  tracked_since = now;
1184 
1185  if (!stored_at) {
1186  log_warn(LD_HIST, "No stored time recorded.");
1187  goto err;
1188  }
1189 
1190  if (line && !strcmp(line, "data"))
1191  ++i;
1192 
1193  n_bogus_times = 0;
1194 
1195  for (; i < smartlist_len(lines); ++i) {
1196  char digest[DIGEST_LEN];
1197  char hexbuf[HEX_DIGEST_LEN+1];
1198  char mtbf_timebuf[ISO_TIME_LEN+1];
1199  char wfu_timebuf[ISO_TIME_LEN+1];
1200  time_t start_of_run = 0;
1201  time_t start_of_downtime = 0;
1202  int have_mtbf = 0, have_wfu = 0;
1203  long wrl = 0;
1204  double trw = 0;
1205  long wt_uptime = 0, total_wt_time = 0;
1206  int n;
1207  or_history_t *hist;
1208  line = smartlist_get(lines, i);
1209  if (!strcmp(line, "."))
1210  break;
1211 
1212  mtbf_timebuf[0] = '\0';
1213  wfu_timebuf[0] = '\0';
1214 
1215  if (format == 1) {
1216  n = tor_sscanf(line, "%40s %ld %lf S=%10s %8s",
1217  hexbuf, &wrl, &trw, mtbf_timebuf, mtbf_timebuf+11);
1218  if (n != 3 && n != 5) {
1219  log_warn(LD_HIST, "Couldn't scan line %s", escaped(line));
1220  continue;
1221  }
1222  have_mtbf = 1;
1223  } else {
1224  // format == 2.
1225  int mtbf_idx, wfu_idx;
1226  if (strcmpstart(line, "R ") || strlen(line) < 2+HEX_DIGEST_LEN)
1227  continue;
1228  strlcpy(hexbuf, line+2, sizeof(hexbuf));
1229  mtbf_idx = find_next_with(lines, i+1, "+MTBF ");
1230  wfu_idx = find_next_with(lines, i+1, "+WFU ");
1231  if (mtbf_idx >= 0) {
1232  const char *mtbfline = smartlist_get(lines, mtbf_idx);
1233  n = tor_sscanf(mtbfline, "+MTBF %lu %lf S=%10s %8s",
1234  &wrl, &trw, mtbf_timebuf, mtbf_timebuf+11);
1235  if (n == 2 || n == 4) {
1236  have_mtbf = 1;
1237  } else {
1238  log_warn(LD_HIST, "Couldn't scan +MTBF line %s",
1239  escaped(mtbfline));
1240  }
1241  }
1242  if (wfu_idx >= 0) {
1243  const char *wfuline = smartlist_get(lines, wfu_idx);
1244  n = tor_sscanf(wfuline, "+WFU %lu %lu S=%10s %8s",
1245  &wt_uptime, &total_wt_time,
1246  wfu_timebuf, wfu_timebuf+11);
1247  if (n == 2 || n == 4) {
1248  have_wfu = 1;
1249  } else {
1250  log_warn(LD_HIST, "Couldn't scan +WFU line %s", escaped(wfuline));
1251  }
1252  }
1253  if (wfu_idx > i)
1254  i = wfu_idx;
1255  if (mtbf_idx > i)
1256  i = mtbf_idx;
1257  }
1258  if (base16_decode(digest, DIGEST_LEN,
1259  hexbuf, HEX_DIGEST_LEN) != DIGEST_LEN) {
1260  log_warn(LD_HIST, "Couldn't hex string %s", escaped(hexbuf));
1261  continue;
1262  }
1263  hist = get_or_history(digest);
1264  if (!hist)
1265  continue;
1266 
1267  if (have_mtbf) {
1268  if (mtbf_timebuf[0]) {
1269  mtbf_timebuf[10] = ' ';
1270  if (parse_possibly_bad_iso_time(mtbf_timebuf, &start_of_run)<0)
1271  log_warn(LD_HIST, "Couldn't parse time %s",
1272  escaped(mtbf_timebuf));
1273  }
1274  hist->start_of_run = correct_time(start_of_run, now, stored_at,
1275  tracked_since);
1276  if (hist->start_of_run < latest_possible_start + wrl)
1277  latest_possible_start = (time_t)(hist->start_of_run - wrl);
1278 
1279  hist->weighted_run_length = wrl;
1280  hist->total_run_weights = trw;
1281  }
1282  if (have_wfu) {
1283  if (wfu_timebuf[0]) {
1284  wfu_timebuf[10] = ' ';
1285  if (parse_possibly_bad_iso_time(wfu_timebuf, &start_of_downtime)<0)
1286  log_warn(LD_HIST, "Couldn't parse time %s", escaped(wfu_timebuf));
1287  }
1288  }
1289  hist->start_of_downtime = correct_time(start_of_downtime, now, stored_at,
1290  tracked_since);
1291  hist->weighted_uptime = wt_uptime;
1292  hist->total_weighted_time = total_wt_time;
1293  }
1294  if (strcmp(line, "."))
1295  log_warn(LD_HIST, "Truncated MTBF file.");
1296 
1297  if (tracked_since < 86400*365) /* Recover from insanely early value. */
1298  tracked_since = latest_possible_start;
1299 
1300  stability_last_downrated = last_downrated;
1301  started_tracking_stability = tracked_since;
1302 
1303  goto done;
1304  err:
1305  r = -1;
1306  done:
1307  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
1308  smartlist_free(lines);
1309  return r;
1310 }
1311 
1312 /*** Exit port statistics ***/
1313 
1314 /* Some constants */
1315 /** To what multiple should byte numbers be rounded up? */
1316 #define EXIT_STATS_ROUND_UP_BYTES 1024
1317 /** To what multiple should stream counts be rounded up? */
1318 #define EXIT_STATS_ROUND_UP_STREAMS 4
1319 /** Number of TCP ports */
1320 #define EXIT_STATS_NUM_PORTS 65536
1321 /** Top n ports that will be included in exit stats. */
1322 #define EXIT_STATS_TOP_N_PORTS 10
1323 
1324 /* The following data structures are arrays and no fancy smartlists or maps,
1325  * so that all write operations can be done in constant time. This comes at
1326  * the price of some memory (1.25 MB) and linear complexity when writing
1327  * stats for measuring relays. */
1328 /** Number of bytes read in current period by exit port */
1329 static uint64_t *exit_bytes_read = NULL;
1330 /** Number of bytes written in current period by exit port */
1331 static uint64_t *exit_bytes_written = NULL;
1332 /** Number of streams opened in current period by exit port */
1333 static uint32_t *exit_streams = NULL;
1334 
1335 /** Start time of exit stats or 0 if we're not collecting exit stats. */
1337 
1338 /** Initialize exit port stats. */
1339 void
1341 {
1343  exit_bytes_read = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t));
1344  exit_bytes_written = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t));
1345  exit_streams = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint32_t));
1346 }
1347 
1348 /** Reset counters for exit port statistics. */
1349 void
1351 {
1353  memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t));
1354  memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t));
1355  memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t));
1356 }
1357 
1358 /** Stop collecting exit port stats in a way that we can re-start doing
1359  * so in rep_hist_exit_stats_init(). */
1360 void
1362 {
1367 }
1368 
1369 /** Helper for qsort: compare two ints. Does not handle overflow properly,
1370  * but works fine for sorting an array of port numbers, which is what we use
1371  * it for. */
1372 static int
1373 compare_int_(const void *x, const void *y)
1374 {
1375  return (*(int*)x - *(int*)y);
1376 }
1377 
1378 /** Return a newly allocated string containing the exit port statistics
1379  * until <b>now</b>, or NULL if we're not collecting exit stats. Caller
1380  * must ensure start_of_exit_stats_interval is in the past. */
1381 char *
1383 {
1384  int i, j, top_elements = 0, cur_min_idx = 0, cur_port;
1385  uint64_t top_bytes[EXIT_STATS_TOP_N_PORTS];
1386  int top_ports[EXIT_STATS_TOP_N_PORTS];
1387  uint64_t cur_bytes = 0, other_read = 0, other_written = 0,
1388  total_read = 0, total_written = 0;
1389  uint32_t total_streams = 0, other_streams = 0;
1390  smartlist_t *written_strings, *read_strings, *streams_strings;
1391  char *written_string, *read_string, *streams_string;
1392  char t[ISO_TIME_LEN+1];
1393  char *result;
1394 
1396  return NULL; /* Not initialized. */
1397 
1399 
1400  /* Go through all ports to find the n ports that saw most written and
1401  * read bytes.
1402  *
1403  * Invariant: at the end of the loop for iteration i,
1404  * total_read is the sum of all exit_bytes_read[0..i]
1405  * total_written is the sum of all exit_bytes_written[0..i]
1406  * total_stream is the sum of all exit_streams[0..i]
1407  *
1408  * top_elements = MAX(EXIT_STATS_TOP_N_PORTS,
1409  * #{j | 0 <= j <= i && volume(i) > 0})
1410  *
1411  * For all 0 <= j < top_elements,
1412  * top_bytes[j] > 0
1413  * 0 <= top_ports[j] <= 65535
1414  * top_bytes[j] = volume(top_ports[j])
1415  *
1416  * There is no j in 0..i and k in 0..top_elements such that:
1417  * volume(j) > top_bytes[k] AND j is not in top_ports[0..top_elements]
1418  *
1419  * There is no j!=cur_min_idx in 0..top_elements such that:
1420  * top_bytes[j] < top_bytes[cur_min_idx]
1421  *
1422  * where volume(x) == exit_bytes_read[x]+exit_bytes_written[x]
1423  *
1424  * Worst case: O(EXIT_STATS_NUM_PORTS * EXIT_STATS_TOP_N_PORTS)
1425  */
1426  for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) {
1427  total_read += exit_bytes_read[i];
1428  total_written += exit_bytes_written[i];
1429  total_streams += exit_streams[i];
1430  cur_bytes = exit_bytes_read[i] + exit_bytes_written[i];
1431  if (cur_bytes == 0) {
1432  continue;
1433  }
1434  if (top_elements < EXIT_STATS_TOP_N_PORTS) {
1435  top_bytes[top_elements] = cur_bytes;
1436  top_ports[top_elements++] = i;
1437  } else if (cur_bytes > top_bytes[cur_min_idx]) {
1438  top_bytes[cur_min_idx] = cur_bytes;
1439  top_ports[cur_min_idx] = i;
1440  } else {
1441  continue;
1442  }
1443  cur_min_idx = 0;
1444  for (j = 1; j < top_elements; j++) {
1445  if (top_bytes[j] < top_bytes[cur_min_idx]) {
1446  cur_min_idx = j;
1447  }
1448  }
1449  }
1450 
1451  /* Add observations of top ports to smartlists. */
1452  written_strings = smartlist_new();
1453  read_strings = smartlist_new();
1454  streams_strings = smartlist_new();
1455  other_read = total_read;
1456  other_written = total_written;
1457  other_streams = total_streams;
1458  /* Sort the ports; this puts them out of sync with top_bytes, but we
1459  * won't be using top_bytes again anyway */
1460  qsort(top_ports, top_elements, sizeof(int), compare_int_);
1461  for (j = 0; j < top_elements; j++) {
1462  cur_port = top_ports[j];
1463  if (exit_bytes_written[cur_port] > 0) {
1464  uint64_t num = round_uint64_to_next_multiple_of(
1465  exit_bytes_written[cur_port],
1467  num /= 1024;
1468  smartlist_add_asprintf(written_strings, "%d=%"PRIu64,
1469  cur_port, (num));
1470  other_written -= exit_bytes_written[cur_port];
1471  }
1472  if (exit_bytes_read[cur_port] > 0) {
1473  uint64_t num = round_uint64_to_next_multiple_of(
1474  exit_bytes_read[cur_port],
1476  num /= 1024;
1477  smartlist_add_asprintf(read_strings, "%d=%"PRIu64,
1478  cur_port, (num));
1479  other_read -= exit_bytes_read[cur_port];
1480  }
1481  if (exit_streams[cur_port] > 0) {
1482  uint32_t num = round_uint32_to_next_multiple_of(
1483  exit_streams[cur_port],
1485  smartlist_add_asprintf(streams_strings, "%d=%u", cur_port, num);
1486  other_streams -= exit_streams[cur_port];
1487  }
1488  }
1489 
1490  /* Add observations of other ports in a single element. */
1491  other_written = round_uint64_to_next_multiple_of(other_written,
1493  other_written /= 1024;
1494  smartlist_add_asprintf(written_strings, "other=%"PRIu64,
1495  (other_written));
1496  other_read = round_uint64_to_next_multiple_of(other_read,
1498  other_read /= 1024;
1499  smartlist_add_asprintf(read_strings, "other=%"PRIu64,
1500  (other_read));
1501  other_streams = round_uint32_to_next_multiple_of(other_streams,
1503  smartlist_add_asprintf(streams_strings, "other=%u", other_streams);
1504 
1505  /* Join all observations in single strings. */
1506  written_string = smartlist_join_strings(written_strings, ",", 0, NULL);
1507  read_string = smartlist_join_strings(read_strings, ",", 0, NULL);
1508  streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL);
1509  SMARTLIST_FOREACH(written_strings, char *, cp, tor_free(cp));
1510  SMARTLIST_FOREACH(read_strings, char *, cp, tor_free(cp));
1511  SMARTLIST_FOREACH(streams_strings, char *, cp, tor_free(cp));
1512  smartlist_free(written_strings);
1513  smartlist_free(read_strings);
1514  smartlist_free(streams_strings);
1515 
1516  /* Put everything together. */
1517  format_iso_time(t, now);
1518  tor_asprintf(&result, "exit-stats-end %s (%d s)\n"
1519  "exit-kibibytes-written %s\n"
1520  "exit-kibibytes-read %s\n"
1521  "exit-streams-opened %s\n",
1522  t, (unsigned) (now - start_of_exit_stats_interval),
1523  written_string,
1524  read_string,
1525  streams_string);
1526  tor_free(written_string);
1527  tor_free(read_string);
1528  tor_free(streams_string);
1529  return result;
1530 }
1531 
1532 /** If 24 hours have passed since the beginning of the current exit port
1533  * stats period, write exit stats to $DATADIR/stats/exit-stats (possibly
1534  * overwriting an existing file) and reset counters. Return when we would
1535  * next want to write exit stats or 0 if we never want to write. */
1536 time_t
1538 {
1539  char *str = NULL;
1540 
1542  return 0; /* Not initialized. */
1543  if (start_of_exit_stats_interval + WRITE_STATS_INTERVAL > now)
1544  goto done; /* Not ready to write. */
1545 
1546  log_info(LD_HIST, "Writing exit port statistics to disk.");
1547 
1548  /* Generate history string. */
1549  str = rep_hist_format_exit_stats(now);
1550 
1551  /* Reset counters. */
1553 
1554  /* Try to write to disk. */
1555  if (!check_or_create_data_subdir("stats")) {
1556  write_to_data_subdir("stats", "exit-stats", str, "exit port statistics");
1557  }
1558 
1559  done:
1560  tor_free(str);
1561  return start_of_exit_stats_interval + WRITE_STATS_INTERVAL;
1562 }
1563 
1564 /** Note that we wrote <b>num_written</b> bytes and read <b>num_read</b>
1565  * bytes to/from an exit connection to <b>port</b>. */
1566 void
1567 rep_hist_note_exit_bytes(uint16_t port, size_t num_written,
1568  size_t num_read)
1569 {
1571  return; /* Not initialized. */
1572  exit_bytes_written[port] += num_written;
1573  exit_bytes_read[port] += num_read;
1574  log_debug(LD_HIST, "Written %lu bytes and read %lu bytes to/from an "
1575  "exit connection to port %d.",
1576  (unsigned long)num_written, (unsigned long)num_read, port);
1577 }
1578 
1579 /** Note that we opened an exit stream to <b>port</b>. */
1580 void
1582 {
1584  return; /* Not initialized. */
1585  exit_streams[port]++;
1586  log_debug(LD_HIST, "Opened exit stream to port %d", port);
1587 }
1588 
1589 /*** cell statistics ***/
1590 
1591 /** Start of the current buffer stats interval or 0 if we're not
1592  * collecting buffer statistics. */
1594 
1595 /** Initialize buffer stats. */
1596 void
1598 {
1600 }
1601 
1602 /** Statistics from a single circuit. Collected when the circuit closes, or
1603  * when we flush statistics to disk. */
1604 typedef struct circ_buffer_stats_t {
1605  /** Average number of cells in the circuit's queue */
1607  /** Average time a cell waits in the queue. */
1609  /** Total number of cells sent over this circuit */
1612 
1613 /** List of circ_buffer_stats_t. */
1615 
1616 /** Remember cell statistics <b>mean_num_cells_in_queue</b>,
1617  * <b>mean_time_cells_in_queue</b>, and <b>processed_cells</b> of a
1618  * circuit. */
1619 void
1620 rep_hist_add_buffer_stats(double mean_num_cells_in_queue,
1621  double mean_time_cells_in_queue, uint32_t processed_cells)
1622 {
1623  circ_buffer_stats_t *stats;
1625  return; /* Not initialized. */
1626  stats = tor_malloc_zero(sizeof(circ_buffer_stats_t));
1627  stats->mean_num_cells_in_queue = mean_num_cells_in_queue;
1628  stats->mean_time_cells_in_queue = mean_time_cells_in_queue;
1629  stats->processed_cells = processed_cells;
1633 }
1634 
1635 /** Remember cell statistics for circuit <b>circ</b> at time
1636  * <b>end_of_interval</b> and reset cell counters in case the circuit
1637  * remains open in the next measurement interval. */
1638 void
1639 rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval)
1640 {
1641  time_t start_of_interval;
1642  int interval_length;
1643  or_circuit_t *orcirc;
1644  double mean_num_cells_in_queue, mean_time_cells_in_queue;
1645  uint32_t processed_cells;
1646  if (CIRCUIT_IS_ORIGIN(circ))
1647  return;
1648  orcirc = TO_OR_CIRCUIT(circ);
1649  if (!orcirc->processed_cells)
1650  return;
1651  start_of_interval = (circ->timestamp_created.tv_sec >
1653  (time_t)circ->timestamp_created.tv_sec :
1655  interval_length = (int) (end_of_interval - start_of_interval);
1656  if (interval_length <= 0)
1657  return;
1658  processed_cells = orcirc->processed_cells;
1659  /* 1000.0 for s -> ms; 2.0 because of app-ward and exit-ward queues */
1660  mean_num_cells_in_queue = (double) orcirc->total_cell_waiting_time /
1661  (double) interval_length / 1000.0 / 2.0;
1662  mean_time_cells_in_queue =
1663  (double) orcirc->total_cell_waiting_time /
1664  (double) orcirc->processed_cells;
1665  orcirc->total_cell_waiting_time = 0;
1666  orcirc->processed_cells = 0;
1667  rep_hist_add_buffer_stats(mean_num_cells_in_queue,
1668  mean_time_cells_in_queue,
1669  processed_cells);
1670 }
1671 
1672 /** Sorting helper: return -1, 1, or 0 based on comparison of two
1673  * circ_buffer_stats_t */
1674 static int
1675 buffer_stats_compare_entries_(const void **_a, const void **_b)
1676 {
1677  const circ_buffer_stats_t *a = *_a, *b = *_b;
1678  if (a->processed_cells < b->processed_cells)
1679  return 1;
1680  else if (a->processed_cells > b->processed_cells)
1681  return -1;
1682  else
1683  return 0;
1684 }
1685 
1686 /** Stop collecting cell stats in a way that we can re-start doing so in
1687  * rep_hist_buffer_stats_init(). */
1688 void
1690 {
1692 }
1693 
1694 /** Clear history of circuit statistics and set the measurement interval
1695  * start to <b>now</b>. */
1696 void
1698 {
1702  stats, tor_free(stats));
1705 }
1706 
1707 /** Return a newly allocated string containing the buffer statistics until
1708  * <b>now</b>, or NULL if we're not collecting buffer stats. Caller must
1709  * ensure start_of_buffer_stats_interval is in the past. */
1710 char *
1712 {
1713 #define SHARES 10
1714  uint64_t processed_cells[SHARES];
1715  uint32_t circs_in_share[SHARES];
1716  int number_of_circuits, i;
1717  double queued_cells[SHARES], time_in_queue[SHARES];
1718  smartlist_t *processed_cells_strings, *queued_cells_strings,
1719  *time_in_queue_strings;
1720  char *processed_cells_string, *queued_cells_string,
1721  *time_in_queue_string;
1722  char t[ISO_TIME_LEN+1];
1723  char *result;
1724 
1726  return NULL; /* Not initialized. */
1727 
1729 
1730  /* Calculate deciles if we saw at least one circuit. */
1731  memset(processed_cells, 0, SHARES * sizeof(uint64_t));
1732  memset(circs_in_share, 0, SHARES * sizeof(uint32_t));
1733  memset(queued_cells, 0, SHARES * sizeof(double));
1734  memset(time_in_queue, 0, SHARES * sizeof(double));
1737  number_of_circuits = smartlist_len(circuits_for_buffer_stats);
1738  if (number_of_circuits > 0) {
1741  i = 0;
1743  circ_buffer_stats_t *, stats)
1744  {
1745  int share = i++ * SHARES / number_of_circuits;
1746  processed_cells[share] += stats->processed_cells;
1747  queued_cells[share] += stats->mean_num_cells_in_queue;
1748  time_in_queue[share] += stats->mean_time_cells_in_queue;
1749  circs_in_share[share]++;
1750  }
1751  SMARTLIST_FOREACH_END(stats);
1752  }
1753 
1754  /* Write deciles to strings. */
1755  processed_cells_strings = smartlist_new();
1756  queued_cells_strings = smartlist_new();
1757  time_in_queue_strings = smartlist_new();
1758  for (i = 0; i < SHARES; i++) {
1759  smartlist_add_asprintf(processed_cells_strings,
1760  "%"PRIu64, !circs_in_share[i] ? 0 :
1761  (processed_cells[i] /
1762  circs_in_share[i]));
1763  }
1764  for (i = 0; i < SHARES; i++) {
1765  smartlist_add_asprintf(queued_cells_strings, "%.2f",
1766  circs_in_share[i] == 0 ? 0.0 :
1767  queued_cells[i] / (double) circs_in_share[i]);
1768  }
1769  for (i = 0; i < SHARES; i++) {
1770  smartlist_add_asprintf(time_in_queue_strings, "%.0f",
1771  circs_in_share[i] == 0 ? 0.0 :
1772  time_in_queue[i] / (double) circs_in_share[i]);
1773  }
1774 
1775  /* Join all observations in single strings. */
1776  processed_cells_string = smartlist_join_strings(processed_cells_strings,
1777  ",", 0, NULL);
1778  queued_cells_string = smartlist_join_strings(queued_cells_strings,
1779  ",", 0, NULL);
1780  time_in_queue_string = smartlist_join_strings(time_in_queue_strings,
1781  ",", 0, NULL);
1782  SMARTLIST_FOREACH(processed_cells_strings, char *, cp, tor_free(cp));
1783  SMARTLIST_FOREACH(queued_cells_strings, char *, cp, tor_free(cp));
1784  SMARTLIST_FOREACH(time_in_queue_strings, char *, cp, tor_free(cp));
1785  smartlist_free(processed_cells_strings);
1786  smartlist_free(queued_cells_strings);
1787  smartlist_free(time_in_queue_strings);
1788 
1789  /* Put everything together. */
1790  format_iso_time(t, now);
1791  tor_asprintf(&result, "cell-stats-end %s (%d s)\n"
1792  "cell-processed-cells %s\n"
1793  "cell-queued-cells %s\n"
1794  "cell-time-in-queue %s\n"
1795  "cell-circuits-per-decile %d\n",
1796  t, (unsigned) (now - start_of_buffer_stats_interval),
1797  processed_cells_string,
1798  queued_cells_string,
1799  time_in_queue_string,
1800  CEIL_DIV(number_of_circuits, SHARES));
1801  tor_free(processed_cells_string);
1802  tor_free(queued_cells_string);
1803  tor_free(time_in_queue_string);
1804  return result;
1805 #undef SHARES
1806 }
1807 
1808 /** If 24 hours have passed since the beginning of the current buffer
1809  * stats period, write buffer stats to $DATADIR/stats/buffer-stats
1810  * (possibly overwriting an existing file) and reset counters. Return
1811  * when we would next want to write buffer stats or 0 if we never want to
1812  * write. */
1813 time_t
1815 {
1816  char *str = NULL;
1817 
1819  return 0; /* Not initialized. */
1820  if (start_of_buffer_stats_interval + WRITE_STATS_INTERVAL > now)
1821  goto done; /* Not ready to write */
1822 
1823  /* Add open circuits to the history. */
1825  rep_hist_buffer_stats_add_circ(circ, now);
1826  }
1827  SMARTLIST_FOREACH_END(circ);
1828 
1829  /* Generate history string. */
1830  str = rep_hist_format_buffer_stats(now);
1831 
1832  /* Reset both buffer history and counters of open circuits. */
1834 
1835  /* Try to write to disk. */
1836  if (!check_or_create_data_subdir("stats")) {
1837  write_to_data_subdir("stats", "buffer-stats", str, "buffer statistics");
1838  }
1839 
1840  done:
1841  tor_free(str);
1842  return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL;
1843 }
1844 
1845 /*** Descriptor serving statistics ***/
1846 
1847 /** Digestmap to track which descriptors were downloaded this stats
1848  * collection interval. It maps descriptor digest to pointers to 1,
1849  * effectively turning this into a list. */
1850 static digestmap_t *served_descs = NULL;
1851 
1852 /** Number of how many descriptors were downloaded in total during this
1853  * interval. */
1854 static unsigned long total_descriptor_downloads;
1855 
1856 /** Start time of served descs stats or 0 if we're not collecting those. */
1858 
1859 /** Initialize descriptor stats. */
1860 void
1862 {
1863  if (served_descs) {
1864  log_warn(LD_BUG, "Called rep_hist_desc_stats_init() when desc stats were "
1865  "already initialized. This is probably harmless.");
1866  return; // Already initialized
1867  }
1868  served_descs = digestmap_new();
1871 }
1872 
1873 /** Reset served descs stats to empty, starting a new interval <b>now</b>. */
1874 static void
1876 {
1879 }
1880 
1881 /** Stop collecting served descs stats, so that rep_hist_desc_stats_init() is
1882  * safe to be called again. */
1883 void
1885 {
1886  digestmap_free(served_descs, NULL);
1887  served_descs = NULL;
1890 }
1891 
1892 /** Helper for rep_hist_desc_stats_write(). Return a newly allocated string
1893  * containing the served desc statistics until now, or NULL if we're not
1894  * collecting served desc stats. Caller must ensure that now is not before
1895  * start_of_served_descs_stats_interval. */
1896 static char *
1898 {
1899  char t[ISO_TIME_LEN+1];
1900  char *result;
1901 
1902  digestmap_iter_t *iter;
1903  const char *key;
1904  void *val;
1905  unsigned size;
1906  int *vals, max = 0, q3 = 0, md = 0, q1 = 0, min = 0;
1907  int n = 0;
1908 
1910  return NULL;
1911 
1912  size = digestmap_size(served_descs);
1913  if (size > 0) {
1914  vals = tor_calloc(size, sizeof(int));
1915  for (iter = digestmap_iter_init(served_descs);
1916  !digestmap_iter_done(iter);
1917  iter = digestmap_iter_next(served_descs, iter)) {
1918  uintptr_t count;
1919  digestmap_iter_get(iter, &key, &val);
1920  count = (uintptr_t)val;
1921  vals[n++] = (int)count;
1922  (void)key;
1923  }
1924  max = find_nth_int(vals, size, size-1);
1925  q3 = find_nth_int(vals, size, (3*size-1)/4);
1926  md = find_nth_int(vals, size, (size-1)/2);
1927  q1 = find_nth_int(vals, size, (size-1)/4);
1928  min = find_nth_int(vals, size, 0);
1929  tor_free(vals);
1930  }
1931 
1932  format_iso_time(t, now);
1933 
1934  tor_asprintf(&result,
1935  "served-descs-stats-end %s (%d s) total=%lu unique=%u "
1936  "max=%d q3=%d md=%d q1=%d min=%d\n",
1937  t,
1938  (unsigned) (now - start_of_served_descs_stats_interval),
1940  size, max, q3, md, q1, min);
1941 
1942  return result;
1943 }
1944 
1945 /** If WRITE_STATS_INTERVAL seconds have passed since the beginning of
1946  * the current served desc stats interval, write the stats to
1947  * $DATADIR/stats/served-desc-stats (possibly appending to an existing file)
1948  * and reset the state for the next interval. Return when we would next want
1949  * to write served desc stats or 0 if we won't want to write. */
1950 time_t
1952 {
1953  char *filename = NULL, *str = NULL;
1954 
1956  return 0; /* We're not collecting stats. */
1957  if (start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL > now)
1958  return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL;
1959 
1960  str = rep_hist_format_desc_stats(now);
1961  tor_assert(str != NULL);
1962 
1963  if (check_or_create_data_subdir("stats") < 0) {
1964  goto done;
1965  }
1966  filename = get_datadir_fname2("stats", "served-desc-stats");
1967  if (append_bytes_to_file(filename, str, strlen(str), 0) < 0)
1968  log_warn(LD_HIST, "Unable to write served descs statistics to disk!");
1969 
1971 
1972  done:
1973  tor_free(filename);
1974  tor_free(str);
1975  return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL;
1976 }
1977 
1978 /** Called to note that we've served a given descriptor (by
1979  * digest). Increments the count of descriptors served, and the number
1980  * of times we've served this descriptor. */
1981 void
1982 rep_hist_note_desc_served(const char * desc)
1983 {
1984  void *val;
1985  uintptr_t count;
1986  if (!served_descs)
1987  return; // We're not collecting stats
1988  val = digestmap_get(served_descs, desc);
1989  count = (uintptr_t)val;
1990  if (count != INT_MAX)
1991  ++count;
1992  digestmap_set(served_descs, desc, (void*)count);
1994 }
1995 
1996 /*** Connection statistics ***/
1997 
1998 /** Internal statistics to track how many requests of each type of
1999  * handshake we've received, and how many we've assigned to cpuworkers.
2000  * Useful for seeing trends in cpu load.
2001  * @{ */
2002 STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0};
2003 STATIC int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0};
2004 STATIC uint64_t onion_handshakes_dropped[MAX_ONION_HANDSHAKE_TYPE+1] = {0};
2005 /**@}*/
2006 
2007 /** A new onionskin (using the <b>type</b> handshake) has arrived. */
2008 void
2010 {
2011  if (type <= MAX_ONION_HANDSHAKE_TYPE)
2013 }
2014 
2015 /** We've sent an onionskin (using the <b>type</b> handshake) to a
2016  * cpuworker. */
2017 void
2019 {
2020  if (type <= MAX_ONION_HANDSHAKE_TYPE)
2021  onion_handshakes_assigned[type]++;
2022 }
2023 
2024 /** We've just drop an onionskin (using the <b>type</b> handshake) due to being
2025  * overloaded. */
2026 void
2028 {
2029  if (type <= MAX_ONION_HANDSHAKE_TYPE)
2030  onion_handshakes_dropped[type]++;
2031 }
2032 
2033 /** Get the circuit handshake value that is requested. */
2034 MOCK_IMPL(int,
2036 {
2037  if (BUG(type > MAX_ONION_HANDSHAKE_TYPE)) {
2038  return 0;
2039  }
2040  return onion_handshakes_requested[type];
2041 }
2042 
2043 /** Get the circuit handshake value that is assigned. */
2044 MOCK_IMPL(int,
2046 {
2047  if (BUG(type > MAX_ONION_HANDSHAKE_TYPE)) {
2048  return 0;
2049  }
2050  return onion_handshakes_assigned[type];
2051 }
2052 
2053 /** Get the circuit handshake value that is dropped. */
2054 MOCK_IMPL(uint64_t,
2056 {
2057  if (BUG(type > MAX_ONION_HANDSHAKE_TYPE)) {
2058  return 0;
2059  }
2060  return onion_handshakes_dropped[type];
2061 }
2062 
2063 /** Log our onionskin statistics since the last time we were called. */
2064 void
2066 {
2067  (void)now;
2068  log_notice(LD_HEARTBEAT, "Circuit handshake stats since last time: "
2069  "%d/%d TAP, %d/%d NTor.",
2070  onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_TAP],
2071  onion_handshakes_requested[ONION_HANDSHAKE_TYPE_TAP],
2072  onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_NTOR],
2073  onion_handshakes_requested[ONION_HANDSHAKE_TYPE_NTOR]);
2074  memset(onion_handshakes_assigned, 0, sizeof(onion_handshakes_assigned));
2076 }
2077 
2078 /* Hidden service statistics section */
2079 
2080 /** Start of the current hidden service stats interval or 0 if we're
2081  * not collecting hidden service statistics. */
2083 
2084 /** Our v2 statistics structure singleton. */
2085 static hs_v2_stats_t *hs_v2_stats = NULL;
2086 
2087 /** HSv2 stats */
2088 
2089 /** Allocate, initialize and return an hs_v2_stats_t structure. */
2090 static hs_v2_stats_t *
2092 {
2093  hs_v2_stats_t *new_hs_v2_stats = tor_malloc_zero(sizeof(hs_v2_stats_t));
2094 
2095  return new_hs_v2_stats;
2096 }
2097 
2098 #define hs_v2_stats_free(val) \
2099  FREE_AND_NULL(hs_v2_stats_t, hs_v2_stats_free_, (val))
2100 
2101 /** Free an hs_v2_stats_t structure. */
2102 static void
2103 hs_v2_stats_free_(hs_v2_stats_t *victim_hs_v2_stats)
2104 {
2105  if (!victim_hs_v2_stats) {
2106  return;
2107  }
2108  tor_free(victim_hs_v2_stats);
2109 }
2110 
2111 /** Clear history of hidden service statistics and set the measurement
2112  * interval start to <b>now</b>. */
2113 static void
2115 {
2116  if (!hs_v2_stats) {
2118  }
2119 
2120  hs_v2_stats->rp_v2_relay_cells_seen = 0;
2121 
2123 }
2124 
2125 /*** HSv3 stats ******/
2126 
2127 /** Start of the current hidden service stats interval or 0 if we're not
2128  * collecting hidden service statistics.
2129  *
2130  * This is particularly important for v3 statistics since this variable
2131  * controls the start time of initial v3 stats collection. It's initialized by
2132  * rep_hist_hs_stats_init() to the next time period start (i.e. 12:00UTC), and
2133  * should_collect_v3_stats() ensures that functions that collect v3 stats do
2134  * not do so sooner than that.
2135  *
2136  * Collecting stats from 12:00UTC to 12:00UTC is extremely important for v3
2137  * stats because rep_hist_hsdir_stored_maybe_new_v3_onion() uses the blinded
2138  * key of each onion service as its double-counting index. Onion services
2139  * rotate their descriptor at around 00:00UTC which means that their blinded
2140  * key also changes around that time. However the precise time that onion
2141  * services rotate their descriptors is actually when they fetch a new
2142  * 00:00UTC consensus and that happens at a random time (e.g. it can even
2143  * happen at 02:00UTC). This means that if we started keeping v3 stats at
2144  * around 00:00UTC we wouldn't be able to tell when onion services change
2145  * their blinded key and hence we would double count an unpredictable amount
2146  * of them (for example, if an onion service fetches the 00:00UTC consensus at
2147  * 01:00UTC it would upload to its old HSDir at 00:45UTC, and then to a
2148  * different HSDir at 01:50UTC).
2149  *
2150  * For this reason, we start collecting statistics at 12:00UTC. This way we
2151  * know that by the time we stop collecting statistics for that time period 24
2152  * hours later, all the onion services have switched to their new blinded
2153  * key. This way we can predict much better how much double counting has been
2154  * performed.
2155  */
2157 
2158 /** Our v3 statistics structure singleton. */
2159 static hs_v3_stats_t *hs_v3_stats = NULL;
2160 
2161 /** Allocate, initialize and return an hs_v3_stats_t structure. */
2162 static hs_v3_stats_t *
2164 {
2165  hs_v3_stats_t *new_hs_v3_stats = tor_malloc_zero(sizeof(hs_v3_stats_t));
2166  new_hs_v3_stats->v3_onions_seen_this_period = digest256map_new();
2167 
2168  return new_hs_v3_stats;
2169 }
2170 
2171 #define hs_v3_stats_free(val) \
2172  FREE_AND_NULL(hs_v3_stats_t, hs_v3_stats_free_, (val))
2173 
2174 /** Free an hs_v3_stats_t structure. */
2175 static void
2176 hs_v3_stats_free_(hs_v3_stats_t *victim_hs_v3_stats)
2177 {
2178  if (!victim_hs_v3_stats) {
2179  return;
2180  }
2181 
2182  digest256map_free(victim_hs_v3_stats->v3_onions_seen_this_period, NULL);
2183  tor_free(victim_hs_v3_stats);
2184 }
2185 
2186 /** Clear history of hidden service statistics and set the measurement
2187  * interval start to <b>now</b>. */
2188 static void
2190 {
2191  if (!hs_v3_stats) {
2193  }
2194 
2195  digest256map_free(hs_v3_stats->v3_onions_seen_this_period, NULL);
2196  hs_v3_stats->v3_onions_seen_this_period = digest256map_new();
2197 
2198  hs_v3_stats->rp_v3_relay_cells_seen = 0;
2199 
2201 }
2202 
2203 /** Return true if it's a good time to collect v3 stats.
2204  *
2205  * v3 stats have a strict stats collection period (from 12:00UTC to 12:00UTC
2206  * on the real network). We don't want to collect statistics if (for example)
2207  * we just booted and it's 03:00UTC; we will wait until 12:00UTC before we
2208  * start collecting statistics to make sure that the final result represents
2209  * the whole collection period. This behavior is controlled by
2210  * rep_hist_hs_stats_init().
2211  */
2212 MOCK_IMPL(STATIC bool,
2214 {
2216 }
2217 
2218 /** We just received a new descriptor with <b>blinded_key</b>. See if we've
2219  * seen this blinded key before, and if not add it to the stats. */
2220 void
2221 rep_hist_hsdir_stored_maybe_new_v3_onion(const uint8_t *blinded_key)
2222 {
2223  /* Return early if we don't collect HSv3 stats, or if it's not yet the time
2224  * to collect them. */
2225  if (!hs_v3_stats || !should_collect_v3_stats()) {
2226  return;
2227  }
2228 
2229  bool seen_before =
2230  !!digest256map_get(hs_v3_stats->v3_onions_seen_this_period,
2231  blinded_key);
2232 
2233  log_info(LD_GENERAL, "Considering v3 descriptor with %s (%sseen before)",
2234  safe_str(hex_str((char*)blinded_key, 32)),
2235  seen_before ? "" : "not ");
2236 
2237  /* Count it if we haven't seen it before. */
2238  if (!seen_before) {
2239  digest256map_set(hs_v3_stats->v3_onions_seen_this_period,
2240  blinded_key, (void*)(uintptr_t)1);
2241  }
2242 }
2243 
2244 /** We saw a new HS relay cell: count it!
2245  * If <b>is_v2</b> is set then it's a v2 RP cell, otherwise it's a v3. */
2246 void
2248 {
2249  log_debug(LD_GENERAL, "New RP cell (%d)", is_v2);
2250 
2251  if (is_v2 && hs_v2_stats) {
2252  hs_v2_stats->rp_v2_relay_cells_seen++;
2253  } else if (!is_v2 && hs_v3_stats && should_collect_v3_stats()) {
2254  hs_v3_stats->rp_v3_relay_cells_seen++;
2255  }
2256 }
2257 
2258 /** Generic HS stats code */
2259 
2260 /** Initialize v2 and v3 hidden service statistics. */
2261 void
2263 {
2264  if (!hs_v2_stats) {
2266  }
2267 
2268  /* Start collecting v2 stats straight away */
2270 
2271  if (!hs_v3_stats) {
2273  }
2274 
2275  /* Start collecting v3 stats at the next 12:00 UTC */
2277 }
2278 
2279 /** Stop collecting hidden service stats in a way that we can re-start
2280  * doing so in rep_hist_buffer_stats_init(). */
2281 void
2283 {
2286 }
2287 
2288 /** Stats reporting code */
2289 
2290 /* The number of cells that are supposed to be hidden from the adversary
2291  * by adding noise from the Laplace distribution. This value, divided by
2292  * EPSILON, is Laplace parameter b. It must be greater than 0. */
2293 #define REND_CELLS_DELTA_F 2048
2294 /* Security parameter for obfuscating number of cells with a value between
2295  * ]0.0, 1.0]. Smaller values obfuscate observations more, but at the same
2296  * time make statistics less usable. */
2297 #define REND_CELLS_EPSILON 0.3
2298 /* The number of cells that are supposed to be hidden from the adversary
2299  * by rounding up to the next multiple of this number. */
2300 #define REND_CELLS_BIN_SIZE 1024
2301 /* The number of service identities that are supposed to be hidden from the
2302  * adversary by adding noise from the Laplace distribution. This value,
2303  * divided by EPSILON, is Laplace parameter b. It must be greater than 0. */
2304 #define ONIONS_SEEN_DELTA_F 8
2305 /* Security parameter for obfuscating number of service identities with a
2306  * value between ]0.0, 1.0]. Smaller values obfuscate observations more, but
2307  * at the same time make statistics less usable. */
2308 #define ONIONS_SEEN_EPSILON 0.3
2309 /* The number of service identities that are supposed to be hidden from
2310  * the adversary by rounding up to the next multiple of this number. */
2311 #define ONIONS_SEEN_BIN_SIZE 8
2312 
2313 /** Allocate and return a string containing hidden service stats that
2314  * are meant to be placed in the extra-info descriptor.
2315  *
2316  * Function works for both v2 and v3 stats depending on <b>is_v3</b>. */
2317 STATIC char *
2318 rep_hist_format_hs_stats(time_t now, bool is_v3)
2319 {
2320  char t[ISO_TIME_LEN+1];
2321  char *hs_stats_string;
2322  int64_t obfuscated_onions_seen, obfuscated_cells_seen;
2323 
2324  uint64_t rp_cells_seen = is_v3 ?
2325  hs_v3_stats->rp_v3_relay_cells_seen : hs_v2_stats->rp_v2_relay_cells_seen;
2326  size_t onions_seen = is_v3 ?
2327  digest256map_size(hs_v3_stats->v3_onions_seen_this_period) : 0;
2328  time_t start_of_hs_stats_interval = is_v3 ?
2330 
2331  uint64_t rounded_cells_seen
2332  = round_uint64_to_next_multiple_of(rp_cells_seen, REND_CELLS_BIN_SIZE);
2333  rounded_cells_seen = MIN(rounded_cells_seen, INT64_MAX);
2334  obfuscated_cells_seen = add_laplace_noise((int64_t)rounded_cells_seen,
2336  REND_CELLS_DELTA_F, REND_CELLS_EPSILON);
2337 
2338  uint64_t rounded_onions_seen =
2339  round_uint64_to_next_multiple_of(onions_seen, ONIONS_SEEN_BIN_SIZE);
2340  rounded_onions_seen = MIN(rounded_onions_seen, INT64_MAX);
2341  obfuscated_onions_seen = add_laplace_noise((int64_t)rounded_onions_seen,
2342  crypto_rand_double(), ONIONS_SEEN_DELTA_F,
2343  ONIONS_SEEN_EPSILON);
2344 
2345  format_iso_time(t, now);
2346  tor_asprintf(&hs_stats_string, "%s %s (%u s)\n"
2347  "%s %"PRId64" delta_f=%d epsilon=%.2f bin_size=%d\n"
2348  "%s %"PRId64" delta_f=%d epsilon=%.2f bin_size=%d\n",
2349  is_v3 ? "hidserv-v3-stats-end" : "hidserv-stats-end",
2350  t, (unsigned) (now - start_of_hs_stats_interval),
2351  is_v3 ?
2352  "hidserv-rend-v3-relayed-cells" : "hidserv-rend-relayed-cells",
2353  obfuscated_cells_seen, REND_CELLS_DELTA_F,
2354  REND_CELLS_EPSILON, REND_CELLS_BIN_SIZE,
2355  is_v3 ? "hidserv-dir-v3-onions-seen" :"hidserv-dir-onions-seen",
2356  obfuscated_onions_seen, ONIONS_SEEN_DELTA_F,
2357  ONIONS_SEEN_EPSILON, ONIONS_SEEN_BIN_SIZE);
2358 
2359  return hs_stats_string;
2360 }
2361 
2362 /** If 24 hours have passed since the beginning of the current HS
2363  * stats period, write buffer stats to $DATADIR/stats/hidserv-v3-stats
2364  * (possibly overwriting an existing file) and reset counters. Return
2365  * when we would next want to write buffer stats or 0 if we never want to
2366  * write. Function works for both v2 and v3 stats depending on <b>is_v3</b>.
2367  */
2368 time_t
2369 rep_hist_hs_stats_write(time_t now, bool is_v3)
2370 {
2371  char *str = NULL;
2372 
2373  time_t start_of_hs_stats_interval = is_v3 ?
2375 
2376  if (!start_of_hs_stats_interval) {
2377  return 0; /* Not initialized. */
2378  }
2379 
2380  if (start_of_hs_stats_interval + WRITE_STATS_INTERVAL > now) {
2381  goto done; /* Not ready to write */
2382  }
2383 
2384  /* Generate history string. */
2385  str = rep_hist_format_hs_stats(now, is_v3);
2386 
2387  /* Reset HS history. */
2388  if (is_v3) {
2390  } else {
2392  }
2393 
2394  /* Try to write to disk. */
2395  if (!check_or_create_data_subdir("stats")) {
2396  write_to_data_subdir("stats",
2397  is_v3 ? "hidserv-v3-stats" : "hidserv-stats",
2398  str, "hidden service stats");
2399  }
2400 
2401  done:
2402  tor_free(str);
2403  return start_of_hs_stats_interval + WRITE_STATS_INTERVAL;
2404 }
2405 
2406 static uint64_t link_proto_count[MAX_LINK_PROTO+1][2];
2407 
2408 /** Note that we negotiated link protocol version <b>link_proto</b>, on
2409  * a connection that started here iff <b>started_here</b> is true.
2410  */
2411 void
2412 rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
2413 {
2414  started_here = !!started_here; /* force to 0 or 1 */
2415  if (link_proto > MAX_LINK_PROTO) {
2416  log_warn(LD_BUG, "Can't log link protocol %u", link_proto);
2417  return;
2418  }
2419 
2420  link_proto_count[link_proto][started_here]++;
2421 }
2422 
2423 /**
2424  * Update the maximum count of total pending channel padding timers
2425  * in this period.
2426  */
2427 void
2428 rep_hist_padding_count_timers(uint64_t num_timers)
2429 {
2430  if (num_timers > padding_current.maximum_chanpad_timers) {
2432  }
2433 }
2434 
2435 /**
2436  * Count a cell that we sent for padding overhead statistics.
2437  *
2438  * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
2439  * counted for PADDING_TYPE_TOTAL.
2440  */
2441 void
2443 {
2444  switch (type) {
2445  case PADDING_TYPE_DROP:
2447  break;
2448  case PADDING_TYPE_CELL:
2450  break;
2451  case PADDING_TYPE_TOTAL:
2453  break;
2456  break;
2459  break;
2460  }
2461 }
2462 
2463 /**
2464  * Count a cell that we've received for padding overhead statistics.
2465  *
2466  * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
2467  * counted for PADDING_TYPE_TOTAL.
2468  */
2469 void
2471 {
2472  switch (type) {
2473  case PADDING_TYPE_DROP:
2475  break;
2476  case PADDING_TYPE_CELL:
2478  break;
2479  case PADDING_TYPE_TOTAL:
2481  break;
2484  break;
2487  break;
2488  }
2489 }
2490 
2491 /**
2492  * Reset our current padding statistics. Called once every 24 hours.
2493  */
2494 void
2496 {
2497  memset(&padding_current, 0, sizeof(padding_current));
2498 }
2499 
2500 /**
2501  * Copy our current cell counts into a structure for listing in our
2502  * extra-info descriptor. Also perform appropriate rounding and redaction.
2503  *
2504  * This function is called once every 24 hours.
2505  */
2506 #define MIN_CELL_COUNTS_TO_PUBLISH 1
2507 #define ROUND_CELL_COUNTS_TO 10000
2508 void
2509 rep_hist_prep_published_padding_counts(time_t now)
2510 {
2512 
2515  memset(&padding_published, 0, sizeof(padding_published));
2516  return;
2517  }
2518 
2520 #define ROUND_AND_SET_COUNT(x) (x) = round_uint64_to_next_multiple_of((x), \
2521  ROUND_CELL_COUNTS_TO)
2522  ROUND_AND_SET_COUNT(padding_published.read_pad_cell_count);
2523  ROUND_AND_SET_COUNT(padding_published.write_pad_cell_count);
2524  ROUND_AND_SET_COUNT(padding_published.read_drop_cell_count);
2525  ROUND_AND_SET_COUNT(padding_published.write_drop_cell_count);
2526  ROUND_AND_SET_COUNT(padding_published.write_cell_count);
2527  ROUND_AND_SET_COUNT(padding_published.read_cell_count);
2528  ROUND_AND_SET_COUNT(padding_published.enabled_read_cell_count);
2529  ROUND_AND_SET_COUNT(padding_published.enabled_read_pad_cell_count);
2530  ROUND_AND_SET_COUNT(padding_published.enabled_write_cell_count);
2531  ROUND_AND_SET_COUNT(padding_published.enabled_write_pad_cell_count);
2532 #undef ROUND_AND_SET_COUNT
2533 }
2534 
2535 /**
2536  * Returns an allocated string for extra-info documents for publishing
2537  * padding statistics from the last 24 hour interval.
2538  */
2539 char *
2541 {
2542  char *result = NULL;
2543 
2546  return NULL;
2547  }
2548 
2549  tor_asprintf(&result, "padding-counts %s (%d s)"
2550  " bin-size=%"PRIu64
2551  " write-drop=%"PRIu64
2552  " write-pad=%"PRIu64
2553  " write-total=%"PRIu64
2554  " read-drop=%"PRIu64
2555  " read-pad=%"PRIu64
2556  " read-total=%"PRIu64
2557  " enabled-read-pad=%"PRIu64
2558  " enabled-read-total=%"PRIu64
2559  " enabled-write-pad=%"PRIu64
2560  " enabled-write-total=%"PRIu64
2561  " max-chanpad-timers=%"PRIu64
2562  "\n",
2565  (uint64_t)ROUND_CELL_COUNTS_TO,
2577  );
2578 
2579  return result;
2580 }
2581 
2582 /** Log a heartbeat message explaining how many connections of each link
2583  * protocol version we have used.
2584  */
2585 void
2587 {
2588  smartlist_t *lines = smartlist_new();
2589 
2590  for (int i = 1; i <= MAX_LINK_PROTO; i++) {
2591  char *line = NULL;
2592  tor_asprintf(&line, "initiated %"PRIu64" and received "
2593  "%"PRIu64" v%d connections", link_proto_count[i][1],
2594  link_proto_count[i][0], i);
2595  smartlist_add(lines, line);
2596  }
2597 
2598  char *log_line = smartlist_join_strings(lines, "; ", 0, NULL);
2599 
2600  log_notice(LD_HEARTBEAT, "Since startup we %s.", log_line);
2601 
2602  SMARTLIST_FOREACH(lines, char *, s, tor_free(s));
2603  smartlist_free(lines);
2604  tor_free(log_line);
2605 }
2606 
2607 /** Free all storage held by the OR/link history caches, by the
2608  * bandwidth history arrays, by the port history, or by statistics . */
2609 void
2611 {
2612  hs_v2_stats_free(hs_v2_stats);
2613  hs_v3_stats_free(hs_v3_stats);
2614  digestmap_free(history_map, free_or_history);
2615 
2621 
2624  tor_free(s));
2625  smartlist_free(circuits_for_buffer_stats);
2627  }
2630 
2631  tor_assert_nonfatal(rephist_total_alloc == 0);
2632  tor_assert_nonfatal_once(rephist_total_num == 0);
2633 }
2634 
2635 #ifdef TOR_UNIT_TESTS
2636 /* only exists for unit tests: get HSv2 stats object */
2637 const hs_v2_stats_t *
2638 rep_hist_get_hs_v2_stats(void)
2639 {
2640  return hs_v2_stats;
2641 }
2642 
2643 /* only exists for unit tests: get HSv2 stats object */
2644 const hs_v3_stats_t *
2645 rep_hist_get_hs_v3_stats(void)
2646 {
2647  return hs_v3_stats;
2648 }
2649 #endif /* defined(TOR_UNIT_TESTS) */
void tor_addr_copy(tor_addr_t *dest, const tor_addr_t *src)
Definition: address.c:933
void tor_addr_make_unspec(tor_addr_t *a)
Definition: address.c:225
int tor_addr_compare(const tor_addr_t *addr1, const tor_addr_t *addr2, tor_addr_comparison_t how)
Definition: address.c:984
int tor_addr_is_null(const tor_addr_t *addr)
Definition: address.c:780
time_t approx_time(void)
Definition: approx_time.c:32
int authdir_mode(const or_options_t *options)
Definition: authmode.c:25
Header file for directory authority mode.
const char * hex_str(const char *from, size_t fromlen)
Definition: binascii.c:34
int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:506
void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:478
smartlist_t * circuit_get_global_list(void)
Definition: circuitlist.c:694
or_circuit_t * TO_OR_CIRCUIT(circuit_t *x)
Definition: circuitlist.c:154
Header file for circuitlist.c.
#define CIRCUIT_IS_ORIGIN(c)
Definition: circuitlist.h:147
const or_options_t * get_options(void)
Definition: config.c:919
int check_or_create_data_subdir(const char *subdir)
Definition: config.c:7139
int write_to_data_subdir(const char *subdir, const char *fname, const char *str, const char *descr)
Definition: config.c:7158
Header file for config.c.
int connection_or_digest_is_known_relay(const char *id_digest)
Header file for connection_or.c.
void conn_stats_free_all(void)
Definition: connstats.c:112
Header for feature/stats/connstats.c.
#define HEX_DIGEST_LEN
Definition: crypto_digest.h:35
Common functions for using (pseudo-)random number generators.
double crypto_rand_double(void)
#define DIGEST_LEN
Definition: digest_sizes.h:20
const char * escaped(const char *s)
Definition: escape.c:126
#define RFTS_IGNORE_MISSING
Definition: files.h:101
int finish_writing_to_file(open_file_t *file_data)
Definition: files.c:465
int append_bytes_to_file(const char *fname, const char *str, size_t len, int bin)
Definition: files.c:554
FILE * start_writing_to_stdio_file(const char *fname, int open_flags, int mode, open_file_t **data_out)
Definition: files.c:398
int abort_writing_to_file(open_file_t *file_data)
Definition: files.c:473
time_t hs_get_start_time_of_next_time_period(time_t now)
Definition: hs_common.c:324
int64_t add_laplace_noise(int64_t signal_, double random_, double delta_f, double epsilon)
Definition: laplace.c:51
Header for laplace.c.
void tor_log(int severity, log_domain_mask_t domain, const char *format,...)
Definition: log.c:590
#define LD_HEARTBEAT
Definition: log.h:103
#define LD_BUG
Definition: log.h:86
#define LD_HIST
Definition: log.h:99
#define LD_GENERAL
Definition: log.h:62
#define tor_free(p)
Definition: malloc.h:52
uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor)
Definition: muldiv.c:35
uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor)
Definition: muldiv.c:50
networkstatus_t * networkstatus_get_latest_consensus(void)
Header file for networkstatus.c.
Networkstatus consensus/vote structure.
const char * node_get_nickname(const node_t *node)
Definition: nodelist.c:1450
const node_t * node_get_by_id(const char *identity_digest)
Definition: nodelist.c:226
Header file for nodelist.c.
Master header file for Tor-specific functionality.
Header for order.c.
long tor_parse_long(const char *s, int base, long min, long max, int *ok, char **next)
Definition: parse_int.c:59
void predicted_ports_free_all(void)
Header file for predict_ports.c.
int tor_asprintf(char **strp, const char *fmt,...)
Definition: printf.c:75
void rep_hist_note_dns_error(int type, uint8_t error)
Definition: rephist.c:325
static int parse_possibly_bad_iso_time(const char *s, time_t *time_out)
Definition: rephist.c:1084
void rep_hist_note_desc_served(const char *desc)
Definition: rephist.c:1982
void rep_hist_buffer_stats_term(void)
Definition: rephist.c:1689
time_t rep_hist_downrate_old_runs(time_t now)
Definition: rephist.c:718
void rep_hist_note_exit_bytes(uint16_t port, size_t num_written, size_t num_read)
Definition: rephist.c:1567
static time_t start_of_hs_v2_stats_interval
Definition: rephist.c:2082
void rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
Definition: rephist.c:2412
void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, const uint16_t at_port, time_t when)
Definition: rephist.c:577
uint64_t rephist_total_alloc
Definition: rephist.c:94
void rep_hist_seen_new_rp_cell(bool is_v2)
Definition: rephist.c:2247
static padding_counts_t padding_current
Definition: rephist.c:172
STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1]
Definition: rephist.c:2002
void rep_hist_desc_stats_term(void)
Definition: rephist.c:1884
static hs_v3_stats_t * hs_v3_stats
Definition: rephist.c:2159
static time_t stability_last_downrated
Definition: rephist.c:180
static long get_total_weighted_time(or_history_t *hist, time_t when)
Definition: rephist.c:785
static digestmap_t * served_descs
Definition: rephist.c:1850
uint64_t rep_hist_get_n_dns_request(int type)
Definition: rephist.c:313
void rep_hist_hs_stats_term(void)
Definition: rephist.c:2282
uint64_t rep_hist_get_circuit_handshake_dropped(uint16_t type)
Definition: rephist.c:2055
static time_t start_of_served_descs_stats_interval
Definition: rephist.c:1857
static dns_stats_t * get_dns_stats_by_type(const int type)
Definition: rephist.c:255
void rep_hist_reset_padding_counts(void)
Definition: rephist.c:2495
double rep_hist_get_stability(const char *id, time_t when)
Definition: rephist.c:839
static bool overload_happened_recently(time_t overload_time, int n_hours)
Definition: rephist.c:391
static void hs_v2_stats_free_(hs_v2_stats_t *victim_hs_v2_stats)
Definition: rephist.c:2103
static uint32_t * exit_streams
Definition: rephist.c:1333
#define EXIT_STATS_TOP_N_PORTS
Definition: rephist.c:1322
int rep_hist_load_mtbf_data(time_t now)
Definition: rephist.c:1125
void rep_hist_note_circuit_handshake_dropped(uint16_t type)
Definition: rephist.c:2027
static or_history_t * get_or_history(const char *id)
Definition: rephist.c:538
void rep_hist_free_all(void)
Definition: rephist.c:2610
static uint64_t * exit_bytes_written
Definition: rephist.c:1331
static overload_stats_t overload_stats
Definition: rephist.c:210
void rep_hist_init(void)
Definition: rephist.c:569
#define EXIT_STATS_ROUND_UP_BYTES
Definition: rephist.c:1316
int rep_hist_get_circuit_handshake_assigned(uint16_t type)
Definition: rephist.c:2045
time_t rep_hist_desc_stats_write(time_t now)
Definition: rephist.c:1951
char * rep_hist_get_overload_stats_lines(void)
Definition: rephist.c:438
void rep_hist_buffer_stats_init(time_t now)
Definition: rephist.c:1597
void rep_hist_exit_stats_init(time_t now)
Definition: rephist.c:1340
static void hs_v3_stats_free_(hs_v3_stats_t *victim_hs_v3_stats)
Definition: rephist.c:2176
void rep_hist_note_exit_stream_opened(uint16_t port)
Definition: rephist.c:1581
static char * rep_hist_format_desc_stats(time_t now)
Definition: rephist.c:1897
double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when)
Definition: rephist.c:851
static hs_v2_stats_t * hs_v2_stats_new(void)
Definition: rephist.c:2091
static uint64_t stats_n_read_limit_reached
Definition: rephist.c:214
void rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval)
Definition: rephist.c:1639
static int find_next_with(smartlist_t *sl, int i, const char *prefix)
Definition: rephist.c:1067
void rep_hist_note_dns_request(int type)
Definition: rephist.c:378
void rep_hist_padding_count_write(padding_type_t type)
Definition: rephist.c:2442
char * rep_hist_get_overload_general_line(void)
Definition: rephist.c:420
long rep_hist_get_uptime(const char *id, time_t when)
Definition: rephist.c:826
static int buffer_stats_compare_entries_(const void **_a, const void **_b)
Definition: rephist.c:1675
static digestmap_t * history_map
Definition: rephist.c:186
long rep_hist_get_weighted_time_known(const char *id, time_t when)
Definition: rephist.c:867
static padding_counts_t padding_published
Definition: rephist.c:176
static uint64_t * exit_bytes_read
Definition: rephist.c:1329
void rep_hist_dump_stats(time_t now, int severity)
Definition: rephist.c:890
static time_t start_of_exit_stats_interval
Definition: rephist.c:1336
static uint64_t stats_n_tcp_exhaustion
Definition: rephist.c:218
void rep_hist_reset_buffer_stats(time_t now)
Definition: rephist.c:1697
static time_t start_of_buffer_stats_interval
Definition: rephist.c:1593
void rep_hist_note_router_unreachable(const char *id, time_t when)
Definition: rephist.c:650
void rep_hist_hs_stats_init(time_t now)
Definition: rephist.c:2262
void rep_hist_hsdir_stored_maybe_new_v3_onion(const uint8_t *blinded_key)
Definition: rephist.c:2221
static int n_bogus_times
Definition: rephist.c:1080
static time_t start_of_hs_v3_stats_interval
Definition: rephist.c:2156
char * rep_hist_format_exit_stats(time_t now)
Definition: rephist.c:1382
void rep_hist_log_link_protocol_counts(void)
Definition: rephist.c:2586
STATIC char * rep_hist_format_hs_stats(time_t now, bool is_v3)
Definition: rephist.c:2318
uint32_t rephist_total_num
Definition: rephist.c:96
char * rep_hist_get_padding_count_lines(void)
Definition: rephist.c:2540
#define SET_TO_START_OF_HOUR(a)
Definition: rephist.c:479
static hs_v3_stats_t * hs_v3_stats_new(void)
Definition: rephist.c:2163
void rep_hist_add_buffer_stats(double mean_num_cells_in_queue, double mean_time_cells_in_queue, uint32_t processed_cells)
Definition: rephist.c:1620
static double get_weighted_fractional_uptime(or_history_t *hist, time_t when)
Definition: rephist.c:799
void rep_hist_note_circuit_handshake_requested(uint16_t type)
Definition: rephist.c:2009
int rep_hist_get_circuit_handshake_requested(uint16_t type)
Definition: rephist.c:2035
void rep_history_clean(time_t before)
Definition: rephist.c:929
static hs_v2_stats_t * hs_v2_stats
Definition: rephist.c:2085
void rep_hist_desc_stats_init(time_t now)
Definition: rephist.c:1861
uint64_t rep_hist_get_n_dns_error(int type, uint8_t error)
Definition: rephist.c:272
static void free_or_history(void *_hist)
Definition: rephist.c:559
#define EXIT_STATS_ROUND_UP_STREAMS
Definition: rephist.c:1318
#define STABILITY_ALPHA
Definition: rephist.c:103
uint64_t rep_hist_get_n_read_limit_reached(void)
Definition: rephist.c:405
static void rep_hist_reset_hs_v2_stats(time_t now)
Definition: rephist.c:2114
#define REND_CELLS_DELTA_F
Definition: rephist.c:2293
static unsigned long total_descriptor_downloads
Definition: rephist.c:1854
uint64_t rep_hist_get_n_tcp_exhaustion(void)
Definition: rephist.c:530
#define STABILITY_EPSILON
Definition: rephist.c:100
static smartlist_t * circuits_for_buffer_stats
Definition: rephist.c:1614
static int compare_int_(const void *x, const void *y)
Definition: rephist.c:1373
static time_t correct_time(time_t t, time_t now, time_t stored_at, time_t started_measuring)
Definition: rephist.c:1105
int rep_hist_record_mtbf_data(time_t now, int missing_means_down)
Definition: rephist.c:962
char * rep_hist_format_buffer_stats(time_t now)
Definition: rephist.c:1711
#define EXIT_STATS_NUM_PORTS
Definition: rephist.c:1320
void rep_hist_reset_exit_stats(time_t now)
Definition: rephist.c:1350
void rep_hist_padding_count_timers(uint64_t num_timers)
Definition: rephist.c:2428
time_t rep_hist_hs_stats_write(time_t now, bool is_v3)
Definition: rephist.c:2369
time_t rep_hist_buffer_stats_write(time_t now)
Definition: rephist.c:1814
void rep_hist_make_router_pessimal(const char *id, time_t when)
Definition: rephist.c:704
void rep_hist_exit_stats_term(void)
Definition: rephist.c:1361
void rep_hist_note_overload(overload_type_t overload)
Definition: rephist.c:485
#define STABILITY_INTERVAL
Definition: rephist.c:105
STATIC bool should_collect_v3_stats(void)
Definition: rephist.c:2213
void rep_hist_log_circuit_handshake_stats(time_t now)
Definition: rephist.c:2065
void rep_hist_note_circuit_handshake_assigned(uint16_t type)
Definition: rephist.c:2018
#define MIN_CELL_COUNTS_TO_PUBLISH
Definition: rephist.c:2506
static void rep_hist_reset_hs_v3_stats(time_t now)
Definition: rephist.c:2189
void rep_hist_note_tcp_exhaustion(void)
Definition: rephist.c:522
void rep_hist_padding_count_read(padding_type_t type)
Definition: rephist.c:2470
int rep_hist_have_measured_enough_stability(void)
Definition: rephist.c:879
static void rep_hist_reset_desc_stats(time_t now)
Definition: rephist.c:1875
uint64_t rep_hist_get_n_write_limit_reached(void)
Definition: rephist.c:412
static dns_stats_t dns_A_stats
Definition: rephist.c:248
time_t rep_hist_exit_stats_write(time_t now)
Definition: rephist.c:1537
static double get_stability(or_history_t *hist, time_t when)
Definition: rephist.c:763
Header file for rephist.c.
padding_type_t
Definition: rephist.h:129
@ PADDING_TYPE_DROP
Definition: rephist.h:131
@ PADDING_TYPE_ENABLED_CELL
Definition: rephist.h:139
@ PADDING_TYPE_TOTAL
Definition: rephist.h:135
@ PADDING_TYPE_ENABLED_TOTAL
Definition: rephist.h:137
@ PADDING_TYPE_CELL
Definition: rephist.h:133
overload_type_t
Definition: rephist.h:155
#define REPHIST_CELL_PADDING_COUNTS_INTERVAL
Definition: rephist.h:143
int tor_sscanf(const char *buf, const char *pattern,...)
Definition: scanf.c:309
void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern,...)
Definition: smartlist.c:36
char * smartlist_join_strings(smartlist_t *sl, const char *join, int terminate, size_t *len_out)
Definition: smartlist.c:279
void smartlist_sort(smartlist_t *sl, int(*compare)(const void **a, const void **b))
Definition: smartlist.c:334
smartlist_t * smartlist_new(void)
void smartlist_add(smartlist_t *sl, void *element)
void smartlist_clear(smartlist_t *sl)
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
#define SMARTLIST_FOREACH(sl, type, var, cmd)
int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep, int flags, int max)
double mean_num_cells_in_queue
Definition: rephist.c:1606
uint32_t processed_cells
Definition: rephist.c:1610
double mean_time_cells_in_queue
Definition: rephist.c:1608
struct timeval timestamp_created
Definition: circuit_st.h:168
Definition: node_st.h:34
uint64_t total_cell_waiting_time
Definition: or_circuit_st.h:79
uint32_t processed_cells
Definition: or_circuit_st.h:74
double total_run_weights
Definition: rephist.c:133
unsigned long weighted_run_length
Definition: rephist.c:128
time_t since
Definition: rephist.c:114
uint16_t last_reached_port
Definition: rephist.c:123
time_t start_of_run
Definition: rephist.c:131
time_t changed
Definition: rephist.c:116
tor_addr_t last_reached_addr
Definition: rephist.c:120
uint64_t BandwidthRate
uint64_t BandwidthBurst
uint64_t maximum_chanpad_timers
Definition: rephist.c:165
uint64_t read_pad_cell_count
Definition: rephist.c:149
uint64_t enabled_read_pad_cell_count
Definition: rephist.c:157
uint64_t write_pad_cell_count
Definition: rephist.c:151
char first_published_at[ISO_TIME_LEN+1]
Definition: rephist.c:167
uint64_t enabled_write_cell_count
Definition: rephist.c:155
uint64_t write_drop_cell_count
Definition: rephist.c:163
uint64_t write_cell_count
Definition: rephist.c:147
uint64_t read_cell_count
Definition: rephist.c:145
uint64_t enabled_read_cell_count
Definition: rephist.c:153
uint64_t enabled_write_pad_cell_count
Definition: rephist.c:159
uint64_t read_drop_cell_count
Definition: rephist.c:161
#define STATIC
Definition: testsupport.h:32
#define MOCK_IMPL(rv, funcname, arglist)
Definition: testsupport.h:133
int parse_iso_time(const char *cp, time_t *t)
Definition: time_fmt.c:392
void format_iso_time(char *buf, time_t t)
Definition: time_fmt.c:295
void format_local_iso_time(char *buf, time_t t)
Definition: time_fmt.c:285
#define tor_assert(expr)
Definition: util_bug.h:102
int strcmpstart(const char *s1, const char *s2)
Definition: util_string.c:215
int tor_digest_is_zero(const char *digest)
Definition: util_string.c:96