Tor  0.4.5.0-alpha-dev
rephist.c
Go to the documentation of this file.
1 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
2  * Copyright (c) 2007-2020, The Tor Project, Inc. */
3 /* See LICENSE for licensing information */
4 
5 /**
6  * \file rephist.c
7  * \brief Basic history and performance-tracking functionality.
8  *
9  * Basic history and performance-tracking functionality to remember
10  * which servers have worked in the past, how much bandwidth we've
11  * been using, which ports we tend to want, and so on; further,
12  * exit port statistics, cell statistics, and connection statistics.
13  *
14  * The history and information tracked in this module could sensibly be
15  * divided into several categories:
16  *
17  * <ul><li>Statistics used by authorities to remember the uptime and
18  * stability information about various relays, including "uptime",
19  * "weighted fractional uptime" and "mean time between failures".
20  *
21  * <li>Predicted ports, used by clients to remember how long it's been
22  * since they opened an exit connection to each given target
23  * port. Clients use this information in order to try to keep circuits
24  * open to exit nodes that can connect to the ports that they care
25  * about. (The predicted ports mechanism also handles predicted circuit
26  * usage that _isn't_ port-specific, such as resolves, internal circuits,
27  * and so on.)
28  *
29  * <li>Public key operation counters, for tracking how many times we've
30  * done each public key operation. (This is unmaintained and we should
31  * remove it.)
32  *
33  * <li>Exit statistics by port, used by exits to keep track of the
34  * number of streams and bytes they've served at each exit port, so they
35  * can generate their exit-kibibytes-{read,written} and
36  * exit-streams-opened statistics.
37  *
38  * <li>Circuit stats, used by relays instances to tract circuit
39  * queue fullness and delay over time, and generate cell-processed-cells,
40  * cell-queued-cells, cell-time-in-queue, and cell-circuits-per-decile
41  * statistics.
42  *
43  * <li>Descriptor serving statistics, used by directory caches to track
44  * how many descriptors they've served.
45  *
46  * <li>Onion handshake statistics, used by relays to count how many
47  * TAP and ntor handshakes they've handled.
48  *
49  * <li>Hidden service statistics, used by relays to count rendezvous
50  * traffic and HSDir-stored descriptors.
51  *
52  * <li>Link protocol statistics, used by relays to count how many times
53  * each link protocol has been used.
54  *
55  * </ul>
56  *
57  * The entry points for this module are scattered throughout the
58  * codebase. Sending data, receiving data, connecting to a relay,
59  * losing a connection to a relay, and so on can all trigger a change in
60  * our current stats. Relays also invoke this module in order to
61  * extract their statistics when building routerinfo and extrainfo
62  * objects in router.c.
63  *
64  * TODO: This module should be broken up.
65  *
66  * (The "rephist" name originally stood for "reputation and history". )
67  **/
68 
69 #define REPHIST_PRIVATE
70 #include "core/or/or.h"
71 #include "app/config/config.h"
72 #include "core/or/circuitlist.h"
73 #include "core/or/connection_or.h"
79 #include "feature/stats/rephist.h"
80 #include "lib/container/order.h"
82 #include "lib/math/laplace.h"
83 
85 #include "core/or/or_circuit_st.h"
86 
87 #ifdef HAVE_FCNTL_H
88 #include <fcntl.h>
89 #endif
90 
91 /** Total number of bytes currently allocated in fields used by rephist.c. */
93 /** Number of or_history_t objects currently allocated. */
94 uint32_t rephist_total_num=0;
95 
96 /** If the total weighted run count of all runs for a router ever falls
97  * below this amount, the router can be treated as having 0 MTBF. */
98 #define STABILITY_EPSILON 0.0001
99 /** Value by which to discount all old intervals for MTBF purposes. This
100  * is compounded every STABILITY_INTERVAL. */
101 #define STABILITY_ALPHA 0.95
102 /** Interval at which to discount all old intervals for MTBF purposes. */
103 #define STABILITY_INTERVAL (12*60*60)
104 /* (This combination of ALPHA, INTERVAL, and EPSILON makes it so that an
105  * interval that just ended counts twice as much as one that ended a week ago,
106  * 20X as much as one that ended a month ago, and routers that have had no
107  * uptime data for about half a year will get forgotten.) */
108 
109 /** History of an OR. */
110 typedef struct or_history_t {
111  /** When did we start tracking this OR? */
112  time_t since;
113  /** When did we most recently note a change to this OR? */
114  time_t changed;
115 
116  /** The address at which we most recently connected to this OR
117  * successfully. */
119 
120  /** The port at which we most recently connected to this OR successfully */
122 
123  /* === For MTBF tracking: */
124  /** Weighted sum total of all times that this router has been online.
125  */
126  unsigned long weighted_run_length;
127  /** If the router is now online (according to stability-checking rules),
128  * when did it come online? */
129  time_t start_of_run;
130  /** Sum of weights for runs in weighted_run_length. */
132  /* === For fractional uptime tracking: */
133  time_t start_of_downtime;
134  unsigned long weighted_uptime;
135  unsigned long total_weighted_time;
136 } or_history_t;
137 
138 /**
139  * This structure holds accounting needed to calculate the padding overhead.
140  */
141 typedef struct padding_counts_t {
142  /** Total number of cells we have received, including padding */
143  uint64_t read_cell_count;
144  /** Total number of cells we have sent, including padding */
146  /** Total number of CELL_PADDING cells we have received */
148  /** Total number of CELL_PADDING cells we have sent */
150  /** Total number of read cells on padding-enabled conns */
152  /** Total number of sent cells on padding-enabled conns */
154  /** Total number of read CELL_PADDING cells on padding-enabled cons */
156  /** Total number of sent CELL_PADDING cells on padding-enabled cons */
158  /** Total number of RELAY_DROP cells we have received */
160  /** Total number of RELAY_DROP cells we have sent */
162  /** The maximum number of padding timers we've seen in 24 hours */
164  /** When did we first copy padding_current into padding_published? */
165  char first_published_at[ISO_TIME_LEN+1];
167 
168 /** Holds the current values of our padding statistics.
169  * It is not published until it is transferred to padding_published. */
171 
172 /** Remains fixed for a 24 hour period, and then is replaced
173  * by a redacted copy of padding_current */
175 
176 /** When did we last multiply all routers' weighted_run_length and
177  * total_run_weights by STABILITY_ALPHA? */
178 static time_t stability_last_downrated = 0;
179 
180 /** */
181 static time_t started_tracking_stability = 0;
182 
183 /** Map from hex OR identity digest to or_history_t. */
184 static digestmap_t *history_map = NULL;
185 
186 /** Return the or_history_t for the OR with identity digest <b>id</b>,
187  * creating it if necessary. */
188 static or_history_t *
189 get_or_history(const char* id)
190 {
191  or_history_t *hist;
192 
193  if (tor_digest_is_zero(id))
194  return NULL;
195 
196  hist = digestmap_get(history_map, id);
197  if (!hist) {
198  hist = tor_malloc_zero(sizeof(or_history_t));
201  hist->since = hist->changed = time(NULL);
203  digestmap_set(history_map, id, hist);
204  }
205  return hist;
206 }
207 
208 /** Helper: free storage held by a single OR history entry. */
209 static void
210 free_or_history(void *_hist)
211 {
212  or_history_t *hist = _hist;
215  tor_free(hist);
216 }
217 
218 /** Initialize the static data structures for tracking history. */
219 void
221 {
222  history_map = digestmap_new();
223 }
224 
225 /** We have just decided that this router with identity digest <b>id</b> is
226  * reachable, meaning we will give it a "Running" flag for the next while. */
227 void
228 rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr,
229  const uint16_t at_port, time_t when)
230 {
231  or_history_t *hist = get_or_history(id);
232  int was_in_run = 1;
233  char tbuf[ISO_TIME_LEN+1];
234  int addr_changed, port_changed;
235 
236  tor_assert(hist);
237  tor_assert((!at_addr && !at_port) || (at_addr && at_port));
238 
239  addr_changed = at_addr && !tor_addr_is_null(&hist->last_reached_addr) &&
240  tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0;
241  port_changed = at_port && hist->last_reached_port &&
242  at_port != hist->last_reached_port;
243 
244  if (!started_tracking_stability)
245  started_tracking_stability = time(NULL);
246  if (!hist->start_of_run) {
247  hist->start_of_run = when;
248  was_in_run = 0;
249  }
250  if (hist->start_of_downtime) {
251  long down_length;
252 
253  format_local_iso_time(tbuf, hist->start_of_downtime);
254  log_info(LD_HIST, "Router %s is now Running; it had been down since %s.",
255  hex_str(id, DIGEST_LEN), tbuf);
256  if (was_in_run)
257  log_info(LD_HIST, " (Paradoxically, it was already Running too.)");
258 
259  down_length = when - hist->start_of_downtime;
260  hist->total_weighted_time += down_length;
261  hist->start_of_downtime = 0;
262  } else if (addr_changed || port_changed) {
263  /* If we're reachable, but the address changed, treat this as some
264  * downtime. */
265  int penalty = get_options()->TestingTorNetwork ? 240 : 3600;
266  networkstatus_t *ns;
267 
268  if ((ns = networkstatus_get_latest_consensus())) {
269  int fresh_interval = (int)(ns->fresh_until - ns->valid_after);
270  int live_interval = (int)(ns->valid_until - ns->valid_after);
271  /* on average, a descriptor addr change takes .5 intervals to make it
272  * into a consensus, and half a liveness period to make it to
273  * clients. */
274  penalty = (int)(fresh_interval + live_interval) / 2;
275  }
276  format_local_iso_time(tbuf, hist->start_of_run);
277  log_info(LD_HIST,"Router %s still seems Running, but its address appears "
278  "to have changed since the last time it was reachable. I'm "
279  "going to treat it as having been down for %d seconds",
280  hex_str(id, DIGEST_LEN), penalty);
281  rep_hist_note_router_unreachable(id, when-penalty);
282  rep_hist_note_router_reachable(id, NULL, 0, when);
283  } else {
284  format_local_iso_time(tbuf, hist->start_of_run);
285  if (was_in_run)
286  log_debug(LD_HIST, "Router %s is still Running; it has been Running "
287  "since %s", hex_str(id, DIGEST_LEN), tbuf);
288  else
289  log_info(LD_HIST,"Router %s is now Running; it was previously untracked",
290  hex_str(id, DIGEST_LEN));
291  }
292  if (at_addr)
293  tor_addr_copy(&hist->last_reached_addr, at_addr);
294  if (at_port)
295  hist->last_reached_port = at_port;
296 }
297 
298 /** We have just decided that this router is unreachable, meaning
299  * we are taking away its "Running" flag. */
300 void
301 rep_hist_note_router_unreachable(const char *id, time_t when)
302 {
303  or_history_t *hist = get_or_history(id);
304  char tbuf[ISO_TIME_LEN+1];
305  int was_running = 0;
306  if (!started_tracking_stability)
307  started_tracking_stability = time(NULL);
308 
309  tor_assert(hist);
310  if (hist->start_of_run) {
311  /*XXXX We could treat failed connections differently from failed
312  * connect attempts. */
313  long run_length = when - hist->start_of_run;
314  format_local_iso_time(tbuf, hist->start_of_run);
315 
316  hist->total_run_weights += 1.0;
317  hist->start_of_run = 0;
318  if (run_length < 0) {
319  unsigned long penalty = -run_length;
320 #define SUBTRACT_CLAMPED(var, penalty) \
321  do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0)
322 
323  SUBTRACT_CLAMPED(hist->weighted_run_length, penalty);
324  SUBTRACT_CLAMPED(hist->weighted_uptime, penalty);
325  } else {
326  hist->weighted_run_length += run_length;
327  hist->weighted_uptime += run_length;
328  hist->total_weighted_time += run_length;
329  }
330  was_running = 1;
331  log_info(LD_HIST, "Router %s is now non-Running: it had previously been "
332  "Running since %s. Its total weighted uptime is %lu/%lu.",
333  hex_str(id, DIGEST_LEN), tbuf, hist->weighted_uptime,
334  hist->total_weighted_time);
335  }
336  if (!hist->start_of_downtime) {
337  hist->start_of_downtime = when;
338 
339  if (!was_running)
340  log_info(LD_HIST, "Router %s is now non-Running; it was previously "
341  "untracked.", hex_str(id, DIGEST_LEN));
342  } else {
343  if (!was_running) {
344  format_local_iso_time(tbuf, hist->start_of_downtime);
345 
346  log_info(LD_HIST, "Router %s is still non-Running; it has been "
347  "non-Running since %s.", hex_str(id, DIGEST_LEN), tbuf);
348  }
349  }
350 }
351 
352 /** Mark a router with ID <b>id</b> as non-Running, and retroactively declare
353  * that it has never been running: give it no stability and no WFU. */
354 void
355 rep_hist_make_router_pessimal(const char *id, time_t when)
356 {
357  or_history_t *hist = get_or_history(id);
358  tor_assert(hist);
359 
361 
362  hist->weighted_run_length = 0;
363  hist->weighted_uptime = 0;
364 }
365 
366 /** Helper: Discount all old MTBF data, if it is time to do so. Return
367  * the time at which we should next discount MTBF data. */
368 time_t
370 {
371  digestmap_iter_t *orhist_it;
372  const char *digest1;
373  or_history_t *hist;
374  void *hist_p;
375  double alpha = 1.0;
376 
377  if (!history_map)
378  history_map = digestmap_new();
383 
384  /* Okay, we should downrate the data. By how much? */
387  alpha *= STABILITY_ALPHA;
388  }
389 
390  log_info(LD_HIST, "Discounting all old stability info by a factor of %f",
391  alpha);
392 
393  /* Multiply every w_r_l, t_r_w pair by alpha. */
394  for (orhist_it = digestmap_iter_init(history_map);
395  !digestmap_iter_done(orhist_it);
396  orhist_it = digestmap_iter_next(history_map,orhist_it)) {
397  digestmap_iter_get(orhist_it, &digest1, &hist_p);
398  hist = hist_p;
399 
400  hist->weighted_run_length =
401  (unsigned long)(hist->weighted_run_length * alpha);
402  hist->total_run_weights *= alpha;
403 
404  hist->weighted_uptime = (unsigned long)(hist->weighted_uptime * alpha);
405  hist->total_weighted_time = (unsigned long)
406  (hist->total_weighted_time * alpha);
407  }
408 
410 }
411 
412 /** Helper: Return the weighted MTBF of the router with history <b>hist</b>. */
413 static double
414 get_stability(or_history_t *hist, time_t when)
415 {
416  long total = hist->weighted_run_length;
417  double total_weights = hist->total_run_weights;
418 
419  if (hist->start_of_run) {
420  /* We're currently in a run. Let total and total_weights hold the values
421  * they would hold if the current run were to end now. */
422  total += (when-hist->start_of_run);
423  total_weights += 1.0;
424  }
425  if (total_weights < STABILITY_EPSILON) {
426  /* Round down to zero, and avoid divide-by-zero. */
427  return 0.0;
428  }
429 
430  return total / total_weights;
431 }
432 
433 /** Return the total amount of time we've been observing, with each run of
434  * time downrated by the appropriate factor. */
435 static long
437 {
438  long total = hist->total_weighted_time;
439  if (hist->start_of_run) {
440  total += (when - hist->start_of_run);
441  } else if (hist->start_of_downtime) {
442  total += (when - hist->start_of_downtime);
443  }
444  return total;
445 }
446 
447 /** Helper: Return the weighted percent-of-time-online of the router with
448  * history <b>hist</b>. */
449 static double
451 {
452  long total = hist->total_weighted_time;
453  long up = hist->weighted_uptime;
454 
455  if (hist->start_of_run) {
456  long run_length = (when - hist->start_of_run);
457  up += run_length;
458  total += run_length;
459  } else if (hist->start_of_downtime) {
460  total += (when - hist->start_of_downtime);
461  }
462 
463  if (!total) {
464  /* Avoid calling anybody's uptime infinity (which should be impossible if
465  * the code is working), or NaN (which can happen for any router we haven't
466  * observed up or down yet). */
467  return 0.0;
468  }
469 
470  return ((double) up) / total;
471 }
472 
473 /** Return how long the router whose identity digest is <b>id</b> has
474  * been reachable. Return 0 if the router is unknown or currently deemed
475  * unreachable. */
476 long
477 rep_hist_get_uptime(const char *id, time_t when)
478 {
479  or_history_t *hist = get_or_history(id);
480  if (!hist)
481  return 0;
482  if (!hist->start_of_run || when < hist->start_of_run)
483  return 0;
484  return when - hist->start_of_run;
485 }
486 
487 /** Return an estimated MTBF for the router whose identity digest is
488  * <b>id</b>. Return 0 if the router is unknown. */
489 double
490 rep_hist_get_stability(const char *id, time_t when)
491 {
492  or_history_t *hist = get_or_history(id);
493  if (!hist)
494  return 0.0;
495 
496  return get_stability(hist, when);
497 }
498 
499 /** Return an estimated percent-of-time-online for the router whose identity
500  * digest is <b>id</b>. Return 0 if the router is unknown. */
501 double
502 rep_hist_get_weighted_fractional_uptime(const char *id, time_t when)
503 {
504  or_history_t *hist = get_or_history(id);
505  if (!hist)
506  return 0.0;
507 
508  return get_weighted_fractional_uptime(hist, when);
509 }
510 
511 /** Return a number representing how long we've known about the router whose
512  * digest is <b>id</b>. Return 0 if the router is unknown.
513  *
514  * Be careful: this measure increases monotonically as we know the router for
515  * longer and longer, but it doesn't increase linearly.
516  */
517 long
518 rep_hist_get_weighted_time_known(const char *id, time_t when)
519 {
520  or_history_t *hist = get_or_history(id);
521  if (!hist)
522  return 0;
523 
524  return get_total_weighted_time(hist, when);
525 }
526 
527 /** Return true if we've been measuring MTBFs for long enough to
528  * pronounce on Stability. */
529 int
531 {
532  /* XXXX++ This doesn't do so well when we change our opinion
533  * as to whether we're tracking router stability. */
534  return started_tracking_stability < time(NULL) - 4*60*60;
535 }
536 
537 /** Log all the reliability data we have remembered, with the chosen
538  * severity.
539  */
540 void
541 rep_hist_dump_stats(time_t now, int severity)
542 {
543  digestmap_iter_t *orhist_it;
544  const char *name1, *digest1;
545  char hexdigest1[HEX_DIGEST_LEN+1];
546  or_history_t *or_history;
547  void *or_history_p;
548  const node_t *node;
549 
550  rep_history_clean(now - get_options()->RephistTrackTime);
551 
552  tor_log(severity, LD_HIST, "--------------- Dumping history information:");
553 
554  for (orhist_it = digestmap_iter_init(history_map);
555  !digestmap_iter_done(orhist_it);
556  orhist_it = digestmap_iter_next(history_map,orhist_it)) {
557  double s;
558  long stability;
559  digestmap_iter_get(orhist_it, &digest1, &or_history_p);
560  or_history = (or_history_t*) or_history_p;
561 
562  if ((node = node_get_by_id(digest1)) && node_get_nickname(node))
563  name1 = node_get_nickname(node);
564  else
565  name1 = "(unknown)";
566  base16_encode(hexdigest1, sizeof(hexdigest1), digest1, DIGEST_LEN);
567  s = get_stability(or_history, now);
568  stability = (long)s;
569  tor_log(severity, LD_HIST,
570  "OR %s [%s]: wmtbf %lu:%02lu:%02lu",
571  name1, hexdigest1,
572  stability/3600, (stability/60)%60, stability%60);
573  }
574 }
575 
576 /** Remove history info for routers/links that haven't changed since
577  * <b>before</b>.
578  */
579 void
580 rep_history_clean(time_t before)
581 {
582  int authority = authdir_mode(get_options());
583  or_history_t *or_history;
584  void *or_history_p;
585  digestmap_iter_t *orhist_it;
586  const char *d1;
587 
588  orhist_it = digestmap_iter_init(history_map);
589  while (!digestmap_iter_done(orhist_it)) {
590  int should_remove;
591  digestmap_iter_get(orhist_it, &d1, &or_history_p);
592  or_history = or_history_p;
593 
594  should_remove = authority ?
595  (or_history->total_run_weights < STABILITY_EPSILON &&
596  !or_history->start_of_run)
597  : (or_history->changed < before);
598  if (should_remove) {
599  orhist_it = digestmap_iter_next_rmv(history_map, orhist_it);
600  free_or_history(or_history);
601  continue;
602  }
603  orhist_it = digestmap_iter_next(history_map, orhist_it);
604  }
605 }
606 
607 /** Write MTBF data to disk. Return 0 on success, negative on failure.
608  *
609  * If <b>missing_means_down</b>, then if we're about to write an entry
610  * that is still considered up but isn't in our routerlist, consider it
611  * to be down. */
612 int
613 rep_hist_record_mtbf_data(time_t now, int missing_means_down)
614 {
615  char time_buf[ISO_TIME_LEN+1];
616 
617  digestmap_iter_t *orhist_it;
618  const char *digest;
619  void *or_history_p;
620  or_history_t *hist;
621  open_file_t *open_file = NULL;
622  FILE *f;
623 
624  {
625  char *filename = get_datadir_fname("router-stability");
626  f = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE|O_TEXT, 0600,
627  &open_file);
628  tor_free(filename);
629  if (!f)
630  return -1;
631  }
632 
633  /* File format is:
634  * FormatLine *KeywordLine Data
635  *
636  * FormatLine = "format 1" NL
637  * KeywordLine = Keyword SP Arguments NL
638  * Data = "data" NL *RouterMTBFLine "." NL
639  * RouterMTBFLine = Fingerprint SP WeightedRunLen SP
640  * TotalRunWeights [SP S=StartRunTime] NL
641  */
642 #define PUT(s) STMT_BEGIN if (fputs((s),f)<0) goto err; STMT_END
643 #define PRINTF(args) STMT_BEGIN if (fprintf args <0) goto err; STMT_END
644 
645  PUT("format 2\n");
646 
647  format_iso_time(time_buf, time(NULL));
648  PRINTF((f, "stored-at %s\n", time_buf));
649 
650  if (started_tracking_stability) {
651  format_iso_time(time_buf, started_tracking_stability);
652  PRINTF((f, "tracked-since %s\n", time_buf));
653  }
656  PRINTF((f, "last-downrated %s\n", time_buf));
657  }
658 
659  PUT("data\n");
660 
661  /* XXX Nick: now bridge auths record this for all routers too.
662  * Should we make them record it only for bridge routers? -RD
663  * Not for 0.2.0. -NM */
664  for (orhist_it = digestmap_iter_init(history_map);
665  !digestmap_iter_done(orhist_it);
666  orhist_it = digestmap_iter_next(history_map,orhist_it)) {
667  char dbuf[HEX_DIGEST_LEN+1];
668  const char *t = NULL;
669  digestmap_iter_get(orhist_it, &digest, &or_history_p);
670  hist = (or_history_t*) or_history_p;
671 
672  base16_encode(dbuf, sizeof(dbuf), digest, DIGEST_LEN);
673 
674  if (missing_means_down && hist->start_of_run &&
676  /* We think this relay is running, but it's not listed in our
677  * consensus. Somehow it fell out without telling us it went
678  * down. Complain and also correct it. */
679  log_info(LD_HIST,
680  "Relay '%s' is listed as up in rephist, but it's not in "
681  "our routerlist. Correcting.", dbuf);
683  }
684 
685  PRINTF((f, "R %s\n", dbuf));
686  if (hist->start_of_run > 0) {
687  format_iso_time(time_buf, hist->start_of_run);
688  t = time_buf;
689  }
690  PRINTF((f, "+MTBF %lu %.5f%s%s\n",
692  t ? " S=" : "", t ? t : ""));
693  t = NULL;
694  if (hist->start_of_downtime > 0) {
695  format_iso_time(time_buf, hist->start_of_downtime);
696  t = time_buf;
697  }
698  PRINTF((f, "+WFU %lu %lu%s%s\n",
699  hist->weighted_uptime, hist->total_weighted_time,
700  t ? " S=" : "", t ? t : ""));
701  }
702 
703  PUT(".\n");
704 
705 #undef PUT
706 #undef PRINTF
707 
708  return finish_writing_to_file(open_file);
709  err:
710  abort_writing_to_file(open_file);
711  return -1;
712 }
713 
714 /** Helper: return the first j >= i such that !strcmpstart(sl[j], prefix) and
715  * such that no line sl[k] with i <= k < j starts with "R ". Return -1 if no
716  * such line exists. */
717 static int
718 find_next_with(smartlist_t *sl, int i, const char *prefix)
719 {
720  for ( ; i < smartlist_len(sl); ++i) {
721  const char *line = smartlist_get(sl, i);
722  if (!strcmpstart(line, prefix))
723  return i;
724  if (!strcmpstart(line, "R "))
725  return -1;
726  }
727  return -1;
728 }
729 
730 /** How many bad times has parse_possibly_bad_iso_time() parsed? */
731 static int n_bogus_times = 0;
732 /** Parse the ISO-formatted time in <b>s</b> into *<b>time_out</b>, but
733  * round any pre-1970 date to Jan 1, 1970. */
734 static int
735 parse_possibly_bad_iso_time(const char *s, time_t *time_out)
736 {
737  int year;
738  char b[5];
739  strlcpy(b, s, sizeof(b));
740  b[4] = '\0';
741  year = (int)tor_parse_long(b, 10, 0, INT_MAX, NULL, NULL);
742  if (year < 1970) {
743  *time_out = 0;
744  ++n_bogus_times;
745  return 0;
746  } else
747  return parse_iso_time(s, time_out);
748 }
749 
750 /** We've read a time <b>t</b> from a file stored at <b>stored_at</b>, which
751  * says we started measuring at <b>started_measuring</b>. Return a new number
752  * that's about as much before <b>now</b> as <b>t</b> was before
753  * <b>stored_at</b>.
754  */
755 static inline time_t
756 correct_time(time_t t, time_t now, time_t stored_at, time_t started_measuring)
757 {
758  if (t < started_measuring - 24*60*60*365)
759  return 0;
760  else if (t < started_measuring)
761  return started_measuring;
762  else if (t > stored_at)
763  return 0;
764  else {
765  long run_length = stored_at - t;
766  t = (time_t)(now - run_length);
767  if (t < started_measuring)
768  t = started_measuring;
769  return t;
770  }
771 }
772 
773 /** Load MTBF data from disk. Returns 0 on success or recoverable error, -1
774  * on failure. */
775 int
777 {
778  /* XXXX won't handle being called while history is already populated. */
779  smartlist_t *lines;
780  const char *line = NULL;
781  int r=0, i;
782  time_t last_downrated = 0, stored_at = 0, tracked_since = 0;
783  time_t latest_possible_start = now;
784  long format = -1;
785 
786  {
787  char *filename = get_datadir_fname("router-stability");
788  char *d = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
789  tor_free(filename);
790  if (!d)
791  return -1;
792  lines = smartlist_new();
793  smartlist_split_string(lines, d, "\n", SPLIT_SKIP_SPACE, 0);
794  tor_free(d);
795  }
796 
797  {
798  const char *firstline;
799  if (smartlist_len(lines)>4) {
800  firstline = smartlist_get(lines, 0);
801  if (!strcmpstart(firstline, "format "))
802  format = tor_parse_long(firstline+strlen("format "),
803  10, -1, LONG_MAX, NULL, NULL);
804  }
805  }
806  if (format != 1 && format != 2) {
807  log_warn(LD_HIST,
808  "Unrecognized format in mtbf history file. Skipping.");
809  goto err;
810  }
811  for (i = 1; i < smartlist_len(lines); ++i) {
812  line = smartlist_get(lines, i);
813  if (!strcmp(line, "data"))
814  break;
815  if (!strcmpstart(line, "last-downrated ")) {
816  if (parse_iso_time(line+strlen("last-downrated "), &last_downrated)<0)
817  log_warn(LD_HIST,"Couldn't parse downrate time in mtbf "
818  "history file.");
819  }
820  if (!strcmpstart(line, "stored-at ")) {
821  if (parse_iso_time(line+strlen("stored-at "), &stored_at)<0)
822  log_warn(LD_HIST,"Couldn't parse stored time in mtbf "
823  "history file.");
824  }
825  if (!strcmpstart(line, "tracked-since ")) {
826  if (parse_iso_time(line+strlen("tracked-since "), &tracked_since)<0)
827  log_warn(LD_HIST,"Couldn't parse started-tracking time in mtbf "
828  "history file.");
829  }
830  }
831  if (last_downrated > now)
832  last_downrated = now;
833  if (tracked_since > now)
834  tracked_since = now;
835 
836  if (!stored_at) {
837  log_warn(LD_HIST, "No stored time recorded.");
838  goto err;
839  }
840 
841  if (line && !strcmp(line, "data"))
842  ++i;
843 
844  n_bogus_times = 0;
845 
846  for (; i < smartlist_len(lines); ++i) {
847  char digest[DIGEST_LEN];
848  char hexbuf[HEX_DIGEST_LEN+1];
849  char mtbf_timebuf[ISO_TIME_LEN+1];
850  char wfu_timebuf[ISO_TIME_LEN+1];
851  time_t start_of_run = 0;
852  time_t start_of_downtime = 0;
853  int have_mtbf = 0, have_wfu = 0;
854  long wrl = 0;
855  double trw = 0;
856  long wt_uptime = 0, total_wt_time = 0;
857  int n;
858  or_history_t *hist;
859  line = smartlist_get(lines, i);
860  if (!strcmp(line, "."))
861  break;
862 
863  mtbf_timebuf[0] = '\0';
864  wfu_timebuf[0] = '\0';
865 
866  if (format == 1) {
867  n = tor_sscanf(line, "%40s %ld %lf S=%10s %8s",
868  hexbuf, &wrl, &trw, mtbf_timebuf, mtbf_timebuf+11);
869  if (n != 3 && n != 5) {
870  log_warn(LD_HIST, "Couldn't scan line %s", escaped(line));
871  continue;
872  }
873  have_mtbf = 1;
874  } else {
875  // format == 2.
876  int mtbf_idx, wfu_idx;
877  if (strcmpstart(line, "R ") || strlen(line) < 2+HEX_DIGEST_LEN)
878  continue;
879  strlcpy(hexbuf, line+2, sizeof(hexbuf));
880  mtbf_idx = find_next_with(lines, i+1, "+MTBF ");
881  wfu_idx = find_next_with(lines, i+1, "+WFU ");
882  if (mtbf_idx >= 0) {
883  const char *mtbfline = smartlist_get(lines, mtbf_idx);
884  n = tor_sscanf(mtbfline, "+MTBF %lu %lf S=%10s %8s",
885  &wrl, &trw, mtbf_timebuf, mtbf_timebuf+11);
886  if (n == 2 || n == 4) {
887  have_mtbf = 1;
888  } else {
889  log_warn(LD_HIST, "Couldn't scan +MTBF line %s",
890  escaped(mtbfline));
891  }
892  }
893  if (wfu_idx >= 0) {
894  const char *wfuline = smartlist_get(lines, wfu_idx);
895  n = tor_sscanf(wfuline, "+WFU %lu %lu S=%10s %8s",
896  &wt_uptime, &total_wt_time,
897  wfu_timebuf, wfu_timebuf+11);
898  if (n == 2 || n == 4) {
899  have_wfu = 1;
900  } else {
901  log_warn(LD_HIST, "Couldn't scan +WFU line %s", escaped(wfuline));
902  }
903  }
904  if (wfu_idx > i)
905  i = wfu_idx;
906  if (mtbf_idx > i)
907  i = mtbf_idx;
908  }
909  if (base16_decode(digest, DIGEST_LEN,
910  hexbuf, HEX_DIGEST_LEN) != DIGEST_LEN) {
911  log_warn(LD_HIST, "Couldn't hex string %s", escaped(hexbuf));
912  continue;
913  }
914  hist = get_or_history(digest);
915  if (!hist)
916  continue;
917 
918  if (have_mtbf) {
919  if (mtbf_timebuf[0]) {
920  mtbf_timebuf[10] = ' ';
921  if (parse_possibly_bad_iso_time(mtbf_timebuf, &start_of_run)<0)
922  log_warn(LD_HIST, "Couldn't parse time %s",
923  escaped(mtbf_timebuf));
924  }
925  hist->start_of_run = correct_time(start_of_run, now, stored_at,
926  tracked_since);
927  if (hist->start_of_run < latest_possible_start + wrl)
928  latest_possible_start = (time_t)(hist->start_of_run - wrl);
929 
930  hist->weighted_run_length = wrl;
931  hist->total_run_weights = trw;
932  }
933  if (have_wfu) {
934  if (wfu_timebuf[0]) {
935  wfu_timebuf[10] = ' ';
936  if (parse_possibly_bad_iso_time(wfu_timebuf, &start_of_downtime)<0)
937  log_warn(LD_HIST, "Couldn't parse time %s", escaped(wfu_timebuf));
938  }
939  }
940  hist->start_of_downtime = correct_time(start_of_downtime, now, stored_at,
941  tracked_since);
942  hist->weighted_uptime = wt_uptime;
943  hist->total_weighted_time = total_wt_time;
944  }
945  if (strcmp(line, "."))
946  log_warn(LD_HIST, "Truncated MTBF file.");
947 
948  if (tracked_since < 86400*365) /* Recover from insanely early value. */
949  tracked_since = latest_possible_start;
950 
951  stability_last_downrated = last_downrated;
952  started_tracking_stability = tracked_since;
953 
954  goto done;
955  err:
956  r = -1;
957  done:
958  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
959  smartlist_free(lines);
960  return r;
961 }
962 
963 /*** Exit port statistics ***/
964 
965 /* Some constants */
966 /** To what multiple should byte numbers be rounded up? */
967 #define EXIT_STATS_ROUND_UP_BYTES 1024
968 /** To what multiple should stream counts be rounded up? */
969 #define EXIT_STATS_ROUND_UP_STREAMS 4
970 /** Number of TCP ports */
971 #define EXIT_STATS_NUM_PORTS 65536
972 /** Top n ports that will be included in exit stats. */
973 #define EXIT_STATS_TOP_N_PORTS 10
974 
975 /* The following data structures are arrays and no fancy smartlists or maps,
976  * so that all write operations can be done in constant time. This comes at
977  * the price of some memory (1.25 MB) and linear complexity when writing
978  * stats for measuring relays. */
979 /** Number of bytes read in current period by exit port */
980 static uint64_t *exit_bytes_read = NULL;
981 /** Number of bytes written in current period by exit port */
982 static uint64_t *exit_bytes_written = NULL;
983 /** Number of streams opened in current period by exit port */
984 static uint32_t *exit_streams = NULL;
985 
986 /** Start time of exit stats or 0 if we're not collecting exit stats. */
988 
989 /** Initialize exit port stats. */
990 void
992 {
994  exit_bytes_read = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t));
995  exit_bytes_written = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t));
996  exit_streams = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint32_t));
997 }
998 
999 /** Reset counters for exit port statistics. */
1000 void
1002 {
1004  memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t));
1005  memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t));
1006  memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t));
1007 }
1008 
1009 /** Stop collecting exit port stats in a way that we can re-start doing
1010  * so in rep_hist_exit_stats_init(). */
1011 void
1013 {
1018 }
1019 
1020 /** Helper for qsort: compare two ints. Does not handle overflow properly,
1021  * but works fine for sorting an array of port numbers, which is what we use
1022  * it for. */
1023 static int
1024 compare_int_(const void *x, const void *y)
1025 {
1026  return (*(int*)x - *(int*)y);
1027 }
1028 
1029 /** Return a newly allocated string containing the exit port statistics
1030  * until <b>now</b>, or NULL if we're not collecting exit stats. Caller
1031  * must ensure start_of_exit_stats_interval is in the past. */
1032 char *
1034 {
1035  int i, j, top_elements = 0, cur_min_idx = 0, cur_port;
1036  uint64_t top_bytes[EXIT_STATS_TOP_N_PORTS];
1037  int top_ports[EXIT_STATS_TOP_N_PORTS];
1038  uint64_t cur_bytes = 0, other_read = 0, other_written = 0,
1039  total_read = 0, total_written = 0;
1040  uint32_t total_streams = 0, other_streams = 0;
1041  smartlist_t *written_strings, *read_strings, *streams_strings;
1042  char *written_string, *read_string, *streams_string;
1043  char t[ISO_TIME_LEN+1];
1044  char *result;
1045 
1047  return NULL; /* Not initialized. */
1048 
1050 
1051  /* Go through all ports to find the n ports that saw most written and
1052  * read bytes.
1053  *
1054  * Invariant: at the end of the loop for iteration i,
1055  * total_read is the sum of all exit_bytes_read[0..i]
1056  * total_written is the sum of all exit_bytes_written[0..i]
1057  * total_stream is the sum of all exit_streams[0..i]
1058  *
1059  * top_elements = MAX(EXIT_STATS_TOP_N_PORTS,
1060  * #{j | 0 <= j <= i && volume(i) > 0})
1061  *
1062  * For all 0 <= j < top_elements,
1063  * top_bytes[j] > 0
1064  * 0 <= top_ports[j] <= 65535
1065  * top_bytes[j] = volume(top_ports[j])
1066  *
1067  * There is no j in 0..i and k in 0..top_elements such that:
1068  * volume(j) > top_bytes[k] AND j is not in top_ports[0..top_elements]
1069  *
1070  * There is no j!=cur_min_idx in 0..top_elements such that:
1071  * top_bytes[j] < top_bytes[cur_min_idx]
1072  *
1073  * where volume(x) == exit_bytes_read[x]+exit_bytes_written[x]
1074  *
1075  * Worst case: O(EXIT_STATS_NUM_PORTS * EXIT_STATS_TOP_N_PORTS)
1076  */
1077  for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) {
1078  total_read += exit_bytes_read[i];
1079  total_written += exit_bytes_written[i];
1080  total_streams += exit_streams[i];
1081  cur_bytes = exit_bytes_read[i] + exit_bytes_written[i];
1082  if (cur_bytes == 0) {
1083  continue;
1084  }
1085  if (top_elements < EXIT_STATS_TOP_N_PORTS) {
1086  top_bytes[top_elements] = cur_bytes;
1087  top_ports[top_elements++] = i;
1088  } else if (cur_bytes > top_bytes[cur_min_idx]) {
1089  top_bytes[cur_min_idx] = cur_bytes;
1090  top_ports[cur_min_idx] = i;
1091  } else {
1092  continue;
1093  }
1094  cur_min_idx = 0;
1095  for (j = 1; j < top_elements; j++) {
1096  if (top_bytes[j] < top_bytes[cur_min_idx]) {
1097  cur_min_idx = j;
1098  }
1099  }
1100  }
1101 
1102  /* Add observations of top ports to smartlists. */
1103  written_strings = smartlist_new();
1104  read_strings = smartlist_new();
1105  streams_strings = smartlist_new();
1106  other_read = total_read;
1107  other_written = total_written;
1108  other_streams = total_streams;
1109  /* Sort the ports; this puts them out of sync with top_bytes, but we
1110  * won't be using top_bytes again anyway */
1111  qsort(top_ports, top_elements, sizeof(int), compare_int_);
1112  for (j = 0; j < top_elements; j++) {
1113  cur_port = top_ports[j];
1114  if (exit_bytes_written[cur_port] > 0) {
1115  uint64_t num = round_uint64_to_next_multiple_of(
1116  exit_bytes_written[cur_port],
1118  num /= 1024;
1119  smartlist_add_asprintf(written_strings, "%d=%"PRIu64,
1120  cur_port, (num));
1121  other_written -= exit_bytes_written[cur_port];
1122  }
1123  if (exit_bytes_read[cur_port] > 0) {
1124  uint64_t num = round_uint64_to_next_multiple_of(
1125  exit_bytes_read[cur_port],
1127  num /= 1024;
1128  smartlist_add_asprintf(read_strings, "%d=%"PRIu64,
1129  cur_port, (num));
1130  other_read -= exit_bytes_read[cur_port];
1131  }
1132  if (exit_streams[cur_port] > 0) {
1133  uint32_t num = round_uint32_to_next_multiple_of(
1134  exit_streams[cur_port],
1136  smartlist_add_asprintf(streams_strings, "%d=%u", cur_port, num);
1137  other_streams -= exit_streams[cur_port];
1138  }
1139  }
1140 
1141  /* Add observations of other ports in a single element. */
1142  other_written = round_uint64_to_next_multiple_of(other_written,
1144  other_written /= 1024;
1145  smartlist_add_asprintf(written_strings, "other=%"PRIu64,
1146  (other_written));
1147  other_read = round_uint64_to_next_multiple_of(other_read,
1149  other_read /= 1024;
1150  smartlist_add_asprintf(read_strings, "other=%"PRIu64,
1151  (other_read));
1152  other_streams = round_uint32_to_next_multiple_of(other_streams,
1154  smartlist_add_asprintf(streams_strings, "other=%u", other_streams);
1155 
1156  /* Join all observations in single strings. */
1157  written_string = smartlist_join_strings(written_strings, ",", 0, NULL);
1158  read_string = smartlist_join_strings(read_strings, ",", 0, NULL);
1159  streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL);
1160  SMARTLIST_FOREACH(written_strings, char *, cp, tor_free(cp));
1161  SMARTLIST_FOREACH(read_strings, char *, cp, tor_free(cp));
1162  SMARTLIST_FOREACH(streams_strings, char *, cp, tor_free(cp));
1163  smartlist_free(written_strings);
1164  smartlist_free(read_strings);
1165  smartlist_free(streams_strings);
1166 
1167  /* Put everything together. */
1168  format_iso_time(t, now);
1169  tor_asprintf(&result, "exit-stats-end %s (%d s)\n"
1170  "exit-kibibytes-written %s\n"
1171  "exit-kibibytes-read %s\n"
1172  "exit-streams-opened %s\n",
1173  t, (unsigned) (now - start_of_exit_stats_interval),
1174  written_string,
1175  read_string,
1176  streams_string);
1177  tor_free(written_string);
1178  tor_free(read_string);
1179  tor_free(streams_string);
1180  return result;
1181 }
1182 
1183 /** If 24 hours have passed since the beginning of the current exit port
1184  * stats period, write exit stats to $DATADIR/stats/exit-stats (possibly
1185  * overwriting an existing file) and reset counters. Return when we would
1186  * next want to write exit stats or 0 if we never want to write. */
1187 time_t
1189 {
1190  char *str = NULL;
1191 
1193  return 0; /* Not initialized. */
1194  if (start_of_exit_stats_interval + WRITE_STATS_INTERVAL > now)
1195  goto done; /* Not ready to write. */
1196 
1197  log_info(LD_HIST, "Writing exit port statistics to disk.");
1198 
1199  /* Generate history string. */
1200  str = rep_hist_format_exit_stats(now);
1201 
1202  /* Reset counters. */
1204 
1205  /* Try to write to disk. */
1206  if (!check_or_create_data_subdir("stats")) {
1207  write_to_data_subdir("stats", "exit-stats", str, "exit port statistics");
1208  }
1209 
1210  done:
1211  tor_free(str);
1212  return start_of_exit_stats_interval + WRITE_STATS_INTERVAL;
1213 }
1214 
1215 /** Note that we wrote <b>num_written</b> bytes and read <b>num_read</b>
1216  * bytes to/from an exit connection to <b>port</b>. */
1217 void
1218 rep_hist_note_exit_bytes(uint16_t port, size_t num_written,
1219  size_t num_read)
1220 {
1222  return; /* Not initialized. */
1223  exit_bytes_written[port] += num_written;
1224  exit_bytes_read[port] += num_read;
1225  log_debug(LD_HIST, "Written %lu bytes and read %lu bytes to/from an "
1226  "exit connection to port %d.",
1227  (unsigned long)num_written, (unsigned long)num_read, port);
1228 }
1229 
1230 /** Note that we opened an exit stream to <b>port</b>. */
1231 void
1233 {
1235  return; /* Not initialized. */
1236  exit_streams[port]++;
1237  log_debug(LD_HIST, "Opened exit stream to port %d", port);
1238 }
1239 
1240 /*** cell statistics ***/
1241 
1242 /** Start of the current buffer stats interval or 0 if we're not
1243  * collecting buffer statistics. */
1245 
1246 /** Initialize buffer stats. */
1247 void
1249 {
1251 }
1252 
1253 /** Statistics from a single circuit. Collected when the circuit closes, or
1254  * when we flush statistics to disk. */
1255 typedef struct circ_buffer_stats_t {
1256  /** Average number of cells in the circuit's queue */
1258  /** Average time a cell waits in the queue. */
1260  /** Total number of cells sent over this circuit */
1263 
1264 /** List of circ_buffer_stats_t. */
1266 
1267 /** Remember cell statistics <b>mean_num_cells_in_queue</b>,
1268  * <b>mean_time_cells_in_queue</b>, and <b>processed_cells</b> of a
1269  * circuit. */
1270 void
1271 rep_hist_add_buffer_stats(double mean_num_cells_in_queue,
1272  double mean_time_cells_in_queue, uint32_t processed_cells)
1273 {
1274  circ_buffer_stats_t *stats;
1276  return; /* Not initialized. */
1277  stats = tor_malloc_zero(sizeof(circ_buffer_stats_t));
1278  stats->mean_num_cells_in_queue = mean_num_cells_in_queue;
1279  stats->mean_time_cells_in_queue = mean_time_cells_in_queue;
1280  stats->processed_cells = processed_cells;
1284 }
1285 
1286 /** Remember cell statistics for circuit <b>circ</b> at time
1287  * <b>end_of_interval</b> and reset cell counters in case the circuit
1288  * remains open in the next measurement interval. */
1289 void
1290 rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval)
1291 {
1292  time_t start_of_interval;
1293  int interval_length;
1294  or_circuit_t *orcirc;
1295  double mean_num_cells_in_queue, mean_time_cells_in_queue;
1296  uint32_t processed_cells;
1297  if (CIRCUIT_IS_ORIGIN(circ))
1298  return;
1299  orcirc = TO_OR_CIRCUIT(circ);
1300  if (!orcirc->processed_cells)
1301  return;
1302  start_of_interval = (circ->timestamp_created.tv_sec >
1304  (time_t)circ->timestamp_created.tv_sec :
1306  interval_length = (int) (end_of_interval - start_of_interval);
1307  if (interval_length <= 0)
1308  return;
1309  processed_cells = orcirc->processed_cells;
1310  /* 1000.0 for s -> ms; 2.0 because of app-ward and exit-ward queues */
1311  mean_num_cells_in_queue = (double) orcirc->total_cell_waiting_time /
1312  (double) interval_length / 1000.0 / 2.0;
1313  mean_time_cells_in_queue =
1314  (double) orcirc->total_cell_waiting_time /
1315  (double) orcirc->processed_cells;
1316  orcirc->total_cell_waiting_time = 0;
1317  orcirc->processed_cells = 0;
1318  rep_hist_add_buffer_stats(mean_num_cells_in_queue,
1319  mean_time_cells_in_queue,
1320  processed_cells);
1321 }
1322 
1323 /** Sorting helper: return -1, 1, or 0 based on comparison of two
1324  * circ_buffer_stats_t */
1325 static int
1326 buffer_stats_compare_entries_(const void **_a, const void **_b)
1327 {
1328  const circ_buffer_stats_t *a = *_a, *b = *_b;
1329  if (a->processed_cells < b->processed_cells)
1330  return 1;
1331  else if (a->processed_cells > b->processed_cells)
1332  return -1;
1333  else
1334  return 0;
1335 }
1336 
1337 /** Stop collecting cell stats in a way that we can re-start doing so in
1338  * rep_hist_buffer_stats_init(). */
1339 void
1341 {
1343 }
1344 
1345 /** Clear history of circuit statistics and set the measurement interval
1346  * start to <b>now</b>. */
1347 void
1349 {
1353  stats, tor_free(stats));
1356 }
1357 
1358 /** Return a newly allocated string containing the buffer statistics until
1359  * <b>now</b>, or NULL if we're not collecting buffer stats. Caller must
1360  * ensure start_of_buffer_stats_interval is in the past. */
1361 char *
1363 {
1364 #define SHARES 10
1365  uint64_t processed_cells[SHARES];
1366  uint32_t circs_in_share[SHARES];
1367  int number_of_circuits, i;
1368  double queued_cells[SHARES], time_in_queue[SHARES];
1369  smartlist_t *processed_cells_strings, *queued_cells_strings,
1370  *time_in_queue_strings;
1371  char *processed_cells_string, *queued_cells_string,
1372  *time_in_queue_string;
1373  char t[ISO_TIME_LEN+1];
1374  char *result;
1375 
1377  return NULL; /* Not initialized. */
1378 
1380 
1381  /* Calculate deciles if we saw at least one circuit. */
1382  memset(processed_cells, 0, SHARES * sizeof(uint64_t));
1383  memset(circs_in_share, 0, SHARES * sizeof(uint32_t));
1384  memset(queued_cells, 0, SHARES * sizeof(double));
1385  memset(time_in_queue, 0, SHARES * sizeof(double));
1388  number_of_circuits = smartlist_len(circuits_for_buffer_stats);
1389  if (number_of_circuits > 0) {
1392  i = 0;
1394  circ_buffer_stats_t *, stats)
1395  {
1396  int share = i++ * SHARES / number_of_circuits;
1397  processed_cells[share] += stats->processed_cells;
1398  queued_cells[share] += stats->mean_num_cells_in_queue;
1399  time_in_queue[share] += stats->mean_time_cells_in_queue;
1400  circs_in_share[share]++;
1401  }
1402  SMARTLIST_FOREACH_END(stats);
1403  }
1404 
1405  /* Write deciles to strings. */
1406  processed_cells_strings = smartlist_new();
1407  queued_cells_strings = smartlist_new();
1408  time_in_queue_strings = smartlist_new();
1409  for (i = 0; i < SHARES; i++) {
1410  smartlist_add_asprintf(processed_cells_strings,
1411  "%"PRIu64, !circs_in_share[i] ? 0 :
1412  (processed_cells[i] /
1413  circs_in_share[i]));
1414  }
1415  for (i = 0; i < SHARES; i++) {
1416  smartlist_add_asprintf(queued_cells_strings, "%.2f",
1417  circs_in_share[i] == 0 ? 0.0 :
1418  queued_cells[i] / (double) circs_in_share[i]);
1419  }
1420  for (i = 0; i < SHARES; i++) {
1421  smartlist_add_asprintf(time_in_queue_strings, "%.0f",
1422  circs_in_share[i] == 0 ? 0.0 :
1423  time_in_queue[i] / (double) circs_in_share[i]);
1424  }
1425 
1426  /* Join all observations in single strings. */
1427  processed_cells_string = smartlist_join_strings(processed_cells_strings,
1428  ",", 0, NULL);
1429  queued_cells_string = smartlist_join_strings(queued_cells_strings,
1430  ",", 0, NULL);
1431  time_in_queue_string = smartlist_join_strings(time_in_queue_strings,
1432  ",", 0, NULL);
1433  SMARTLIST_FOREACH(processed_cells_strings, char *, cp, tor_free(cp));
1434  SMARTLIST_FOREACH(queued_cells_strings, char *, cp, tor_free(cp));
1435  SMARTLIST_FOREACH(time_in_queue_strings, char *, cp, tor_free(cp));
1436  smartlist_free(processed_cells_strings);
1437  smartlist_free(queued_cells_strings);
1438  smartlist_free(time_in_queue_strings);
1439 
1440  /* Put everything together. */
1441  format_iso_time(t, now);
1442  tor_asprintf(&result, "cell-stats-end %s (%d s)\n"
1443  "cell-processed-cells %s\n"
1444  "cell-queued-cells %s\n"
1445  "cell-time-in-queue %s\n"
1446  "cell-circuits-per-decile %d\n",
1447  t, (unsigned) (now - start_of_buffer_stats_interval),
1448  processed_cells_string,
1449  queued_cells_string,
1450  time_in_queue_string,
1451  CEIL_DIV(number_of_circuits, SHARES));
1452  tor_free(processed_cells_string);
1453  tor_free(queued_cells_string);
1454  tor_free(time_in_queue_string);
1455  return result;
1456 #undef SHARES
1457 }
1458 
1459 /** If 24 hours have passed since the beginning of the current buffer
1460  * stats period, write buffer stats to $DATADIR/stats/buffer-stats
1461  * (possibly overwriting an existing file) and reset counters. Return
1462  * when we would next want to write buffer stats or 0 if we never want to
1463  * write. */
1464 time_t
1466 {
1467  char *str = NULL;
1468 
1470  return 0; /* Not initialized. */
1471  if (start_of_buffer_stats_interval + WRITE_STATS_INTERVAL > now)
1472  goto done; /* Not ready to write */
1473 
1474  /* Add open circuits to the history. */
1476  rep_hist_buffer_stats_add_circ(circ, now);
1477  }
1478  SMARTLIST_FOREACH_END(circ);
1479 
1480  /* Generate history string. */
1481  str = rep_hist_format_buffer_stats(now);
1482 
1483  /* Reset both buffer history and counters of open circuits. */
1485 
1486  /* Try to write to disk. */
1487  if (!check_or_create_data_subdir("stats")) {
1488  write_to_data_subdir("stats", "buffer-stats", str, "buffer statistics");
1489  }
1490 
1491  done:
1492  tor_free(str);
1493  return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL;
1494 }
1495 
1496 /*** Descriptor serving statistics ***/
1497 
1498 /** Digestmap to track which descriptors were downloaded this stats
1499  * collection interval. It maps descriptor digest to pointers to 1,
1500  * effectively turning this into a list. */
1501 static digestmap_t *served_descs = NULL;
1502 
1503 /** Number of how many descriptors were downloaded in total during this
1504  * interval. */
1505 static unsigned long total_descriptor_downloads;
1506 
1507 /** Start time of served descs stats or 0 if we're not collecting those. */
1509 
1510 /** Initialize descriptor stats. */
1511 void
1513 {
1514  if (served_descs) {
1515  log_warn(LD_BUG, "Called rep_hist_desc_stats_init() when desc stats were "
1516  "already initialized. This is probably harmless.");
1517  return; // Already initialized
1518  }
1519  served_descs = digestmap_new();
1522 }
1523 
1524 /** Reset served descs stats to empty, starting a new interval <b>now</b>. */
1525 static void
1527 {
1530 }
1531 
1532 /** Stop collecting served descs stats, so that rep_hist_desc_stats_init() is
1533  * safe to be called again. */
1534 void
1536 {
1537  digestmap_free(served_descs, NULL);
1538  served_descs = NULL;
1541 }
1542 
1543 /** Helper for rep_hist_desc_stats_write(). Return a newly allocated string
1544  * containing the served desc statistics until now, or NULL if we're not
1545  * collecting served desc stats. Caller must ensure that now is not before
1546  * start_of_served_descs_stats_interval. */
1547 static char *
1549 {
1550  char t[ISO_TIME_LEN+1];
1551  char *result;
1552 
1553  digestmap_iter_t *iter;
1554  const char *key;
1555  void *val;
1556  unsigned size;
1557  int *vals, max = 0, q3 = 0, md = 0, q1 = 0, min = 0;
1558  int n = 0;
1559 
1561  return NULL;
1562 
1563  size = digestmap_size(served_descs);
1564  if (size > 0) {
1565  vals = tor_calloc(size, sizeof(int));
1566  for (iter = digestmap_iter_init(served_descs);
1567  !digestmap_iter_done(iter);
1568  iter = digestmap_iter_next(served_descs, iter)) {
1569  uintptr_t count;
1570  digestmap_iter_get(iter, &key, &val);
1571  count = (uintptr_t)val;
1572  vals[n++] = (int)count;
1573  (void)key;
1574  }
1575  max = find_nth_int(vals, size, size-1);
1576  q3 = find_nth_int(vals, size, (3*size-1)/4);
1577  md = find_nth_int(vals, size, (size-1)/2);
1578  q1 = find_nth_int(vals, size, (size-1)/4);
1579  min = find_nth_int(vals, size, 0);
1580  tor_free(vals);
1581  }
1582 
1583  format_iso_time(t, now);
1584 
1585  tor_asprintf(&result,
1586  "served-descs-stats-end %s (%d s) total=%lu unique=%u "
1587  "max=%d q3=%d md=%d q1=%d min=%d\n",
1588  t,
1589  (unsigned) (now - start_of_served_descs_stats_interval),
1591  size, max, q3, md, q1, min);
1592 
1593  return result;
1594 }
1595 
1596 /** If WRITE_STATS_INTERVAL seconds have passed since the beginning of
1597  * the current served desc stats interval, write the stats to
1598  * $DATADIR/stats/served-desc-stats (possibly appending to an existing file)
1599  * and reset the state for the next interval. Return when we would next want
1600  * to write served desc stats or 0 if we won't want to write. */
1601 time_t
1603 {
1604  char *filename = NULL, *str = NULL;
1605 
1607  return 0; /* We're not collecting stats. */
1608  if (start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL > now)
1609  return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL;
1610 
1611  str = rep_hist_format_desc_stats(now);
1612  tor_assert(str != NULL);
1613 
1614  if (check_or_create_data_subdir("stats") < 0) {
1615  goto done;
1616  }
1617  filename = get_datadir_fname2("stats", "served-desc-stats");
1618  if (append_bytes_to_file(filename, str, strlen(str), 0) < 0)
1619  log_warn(LD_HIST, "Unable to write served descs statistics to disk!");
1620 
1622 
1623  done:
1624  tor_free(filename);
1625  tor_free(str);
1626  return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL;
1627 }
1628 
1629 /** Called to note that we've served a given descriptor (by
1630  * digest). Increments the count of descriptors served, and the number
1631  * of times we've served this descriptor. */
1632 void
1633 rep_hist_note_desc_served(const char * desc)
1634 {
1635  void *val;
1636  uintptr_t count;
1637  if (!served_descs)
1638  return; // We're not collecting stats
1639  val = digestmap_get(served_descs, desc);
1640  count = (uintptr_t)val;
1641  if (count != INT_MAX)
1642  ++count;
1643  digestmap_set(served_descs, desc, (void*)count);
1645 }
1646 
1647 /*** Connection statistics ***/
1648 
1649 /** Internal statistics to track how many requests of each type of
1650  * handshake we've received, and how many we've assigned to cpuworkers.
1651  * Useful for seeing trends in cpu load.
1652  * @{ */
1653 STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0};
1654 STATIC int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0};
1655 /**@}*/
1656 
1657 /** A new onionskin (using the <b>type</b> handshake) has arrived. */
1658 void
1660 {
1661  if (type <= MAX_ONION_HANDSHAKE_TYPE)
1663 }
1664 
1665 /** We've sent an onionskin (using the <b>type</b> handshake) to a
1666  * cpuworker. */
1667 void
1669 {
1670  if (type <= MAX_ONION_HANDSHAKE_TYPE)
1671  onion_handshakes_assigned[type]++;
1672 }
1673 
1674 /** Get the circuit handshake value that is requested. */
1675 MOCK_IMPL(int,
1677 {
1678  if (BUG(type > MAX_ONION_HANDSHAKE_TYPE)) {
1679  return 0;
1680  }
1681  return onion_handshakes_requested[type];
1682 }
1683 
1684 /** Get the circuit handshake value that is assigned. */
1685 MOCK_IMPL(int,
1687 {
1688  if (BUG(type > MAX_ONION_HANDSHAKE_TYPE)) {
1689  return 0;
1690  }
1691  return onion_handshakes_assigned[type];
1692 }
1693 
1694 /** Log our onionskin statistics since the last time we were called. */
1695 void
1697 {
1698  (void)now;
1699  log_notice(LD_HEARTBEAT, "Circuit handshake stats since last time: "
1700  "%d/%d TAP, %d/%d NTor.",
1701  onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_TAP],
1702  onion_handshakes_requested[ONION_HANDSHAKE_TYPE_TAP],
1703  onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_NTOR],
1704  onion_handshakes_requested[ONION_HANDSHAKE_TYPE_NTOR]);
1705  memset(onion_handshakes_assigned, 0, sizeof(onion_handshakes_assigned));
1707 }
1708 
1709 /* Hidden service statistics section */
1710 
1711 /** Start of the current hidden service stats interval or 0 if we're
1712  * not collecting hidden service statistics. */
1714 
1715 /** Carries the various hidden service statistics, and any other
1716  * information needed. */
1717 typedef struct hs_stats_t {
1718  /** How many relay cells have we seen as rendezvous points? */
1720 
1721  /** Set of unique public key digests we've seen this stat period
1722  * (could also be implemented as sorted smartlist). */
1724 } hs_stats_t;
1725 
1726 /** Our statistics structure singleton. */
1727 static hs_stats_t *hs_stats = NULL;
1728 
1729 /** Allocate, initialize and return an hs_stats_t structure. */
1730 static hs_stats_t *
1732 {
1733  hs_stats_t *new_hs_stats = tor_malloc_zero(sizeof(hs_stats_t));
1734  new_hs_stats->onions_seen_this_period = digestmap_new();
1735 
1736  return new_hs_stats;
1737 }
1738 
1739 #define hs_stats_free(val) \
1740  FREE_AND_NULL(hs_stats_t, hs_stats_free_, (val))
1741 
1742 /** Free an hs_stats_t structure. */
1743 static void
1744 hs_stats_free_(hs_stats_t *victim_hs_stats)
1745 {
1746  if (!victim_hs_stats) {
1747  return;
1748  }
1749 
1750  digestmap_free(victim_hs_stats->onions_seen_this_period, NULL);
1751  tor_free(victim_hs_stats);
1752 }
1753 
1754 /** Initialize hidden service statistics. */
1755 void
1757 {
1758  if (!hs_stats) {
1759  hs_stats = hs_stats_new();
1760  }
1761 
1763 }
1764 
1765 /** Clear history of hidden service statistics and set the measurement
1766  * interval start to <b>now</b>. */
1767 static void
1769 {
1770  if (!hs_stats) {
1771  hs_stats = hs_stats_new();
1772  }
1773 
1775 
1776  digestmap_free(hs_stats->onions_seen_this_period, NULL);
1777  hs_stats->onions_seen_this_period = digestmap_new();
1778 
1780 }
1781 
1782 /** Stop collecting hidden service stats in a way that we can re-start
1783  * doing so in rep_hist_buffer_stats_init(). */
1784 void
1786 {
1788 }
1789 
1790 /** We saw a new HS relay cell, Count it! */
1791 void
1793 {
1794  if (!hs_stats) {
1795  return; // We're not collecting stats
1796  }
1797 
1799 }
1800 
1801 /** As HSDirs, we saw another hidden service with public key
1802  * <b>pubkey</b>. Check whether we have counted it before, if not
1803  * count it now! */
1804 void
1806 {
1807  char pubkey_hash[DIGEST_LEN];
1808 
1809  if (!hs_stats) {
1810  return; // We're not collecting stats
1811  }
1812 
1813  /* Get the digest of the pubkey which will be used to detect whether
1814  we've seen this hidden service before or not. */
1815  if (crypto_pk_get_digest(pubkey, pubkey_hash) < 0) {
1816  /* This fail should not happen; key has been validated by
1817  descriptor parsing code first. */
1818  return;
1819  }
1820 
1821  /* Check if this is the first time we've seen this hidden
1822  service. If it is, count it as new. */
1823  if (!digestmap_get(hs_stats->onions_seen_this_period,
1824  pubkey_hash)) {
1825  digestmap_set(hs_stats->onions_seen_this_period,
1826  pubkey_hash, (void*)(uintptr_t)1);
1827  }
1828 }
1829 
1830 /* The number of cells that are supposed to be hidden from the adversary
1831  * by adding noise from the Laplace distribution. This value, divided by
1832  * EPSILON, is Laplace parameter b. It must be greather than 0. */
1833 #define REND_CELLS_DELTA_F 2048
1834 /* Security parameter for obfuscating number of cells with a value between
1835  * ]0.0, 1.0]. Smaller values obfuscate observations more, but at the same
1836  * time make statistics less usable. */
1837 #define REND_CELLS_EPSILON 0.3
1838 /* The number of cells that are supposed to be hidden from the adversary
1839  * by rounding up to the next multiple of this number. */
1840 #define REND_CELLS_BIN_SIZE 1024
1841 /* The number of service identities that are supposed to be hidden from the
1842  * adversary by adding noise from the Laplace distribution. This value,
1843  * divided by EPSILON, is Laplace parameter b. It must be greater than 0. */
1844 #define ONIONS_SEEN_DELTA_F 8
1845 /* Security parameter for obfuscating number of service identities with a
1846  * value between ]0.0, 1.0]. Smaller values obfuscate observations more, but
1847  * at the same time make statistics less usable. */
1848 #define ONIONS_SEEN_EPSILON 0.3
1849 /* The number of service identities that are supposed to be hidden from
1850  * the adversary by rounding up to the next multiple of this number. */
1851 #define ONIONS_SEEN_BIN_SIZE 8
1852 
1853 /** Allocate and return a string containing hidden service stats that
1854  * are meant to be placed in the extra-info descriptor. */
1855 static char *
1857 {
1858  char t[ISO_TIME_LEN+1];
1859  char *hs_stats_string;
1860  int64_t obfuscated_cells_seen;
1861  int64_t obfuscated_onions_seen;
1862 
1863  uint64_t rounded_cells_seen
1865  REND_CELLS_BIN_SIZE);
1866  rounded_cells_seen = MIN(rounded_cells_seen, INT64_MAX);
1867  obfuscated_cells_seen = add_laplace_noise((int64_t)rounded_cells_seen,
1869  REND_CELLS_DELTA_F, REND_CELLS_EPSILON);
1870 
1871  uint64_t rounded_onions_seen =
1872  round_uint64_to_next_multiple_of((size_t)digestmap_size(
1874  ONIONS_SEEN_BIN_SIZE);
1875  rounded_onions_seen = MIN(rounded_onions_seen, INT64_MAX);
1876  obfuscated_onions_seen = add_laplace_noise((int64_t)rounded_onions_seen,
1877  crypto_rand_double(), ONIONS_SEEN_DELTA_F,
1878  ONIONS_SEEN_EPSILON);
1879 
1880  format_iso_time(t, now);
1881  tor_asprintf(&hs_stats_string, "hidserv-stats-end %s (%d s)\n"
1882  "hidserv-rend-relayed-cells %"PRId64" delta_f=%d "
1883  "epsilon=%.2f bin_size=%d\n"
1884  "hidserv-dir-onions-seen %"PRId64" delta_f=%d "
1885  "epsilon=%.2f bin_size=%d\n",
1886  t, (unsigned) (now - start_of_hs_stats_interval),
1887  (obfuscated_cells_seen), REND_CELLS_DELTA_F,
1888  REND_CELLS_EPSILON, REND_CELLS_BIN_SIZE,
1889  (obfuscated_onions_seen),
1890  ONIONS_SEEN_DELTA_F,
1891  ONIONS_SEEN_EPSILON, ONIONS_SEEN_BIN_SIZE);
1892 
1893  return hs_stats_string;
1894 }
1895 
1896 /** If 24 hours have passed since the beginning of the current HS
1897  * stats period, write buffer stats to $DATADIR/stats/hidserv-stats
1898  * (possibly overwriting an existing file) and reset counters. Return
1899  * when we would next want to write buffer stats or 0 if we never want to
1900  * write. */
1901 time_t
1903 {
1904  char *str = NULL;
1905 
1907  return 0; /* Not initialized. */
1908  }
1909 
1910  if (start_of_hs_stats_interval + WRITE_STATS_INTERVAL > now) {
1911  goto done; /* Not ready to write */
1912  }
1913 
1914  /* Generate history string. */
1915  str = rep_hist_format_hs_stats(now);
1916 
1917  /* Reset HS history. */
1919 
1920  /* Try to write to disk. */
1921  if (!check_or_create_data_subdir("stats")) {
1922  write_to_data_subdir("stats", "hidserv-stats", str,
1923  "hidden service stats");
1924  }
1925 
1926  done:
1927  tor_free(str);
1928  return start_of_hs_stats_interval + WRITE_STATS_INTERVAL;
1929 }
1930 
1931 static uint64_t link_proto_count[MAX_LINK_PROTO+1][2];
1932 
1933 /** Note that we negotiated link protocol version <b>link_proto</b>, on
1934  * a connection that started here iff <b>started_here</b> is true.
1935  */
1936 void
1937 rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
1938 {
1939  started_here = !!started_here; /* force to 0 or 1 */
1940  if (link_proto > MAX_LINK_PROTO) {
1941  log_warn(LD_BUG, "Can't log link protocol %u", link_proto);
1942  return;
1943  }
1944 
1945  link_proto_count[link_proto][started_here]++;
1946 }
1947 
1948 /**
1949  * Update the maximum count of total pending channel padding timers
1950  * in this period.
1951  */
1952 void
1953 rep_hist_padding_count_timers(uint64_t num_timers)
1954 {
1955  if (num_timers > padding_current.maximum_chanpad_timers) {
1957  }
1958 }
1959 
1960 /**
1961  * Count a cell that we sent for padding overhead statistics.
1962  *
1963  * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
1964  * counted for PADDING_TYPE_TOTAL.
1965  */
1966 void
1968 {
1969  switch (type) {
1970  case PADDING_TYPE_DROP:
1972  break;
1973  case PADDING_TYPE_CELL:
1975  break;
1976  case PADDING_TYPE_TOTAL:
1978  break;
1981  break;
1984  break;
1985  }
1986 }
1987 
1988 /**
1989  * Count a cell that we've received for padding overhead statistics.
1990  *
1991  * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
1992  * counted for PADDING_TYPE_TOTAL.
1993  */
1994 void
1996 {
1997  switch (type) {
1998  case PADDING_TYPE_DROP:
2000  break;
2001  case PADDING_TYPE_CELL:
2003  break;
2004  case PADDING_TYPE_TOTAL:
2006  break;
2009  break;
2012  break;
2013  }
2014 }
2015 
2016 /**
2017  * Reset our current padding statistics. Called once every 24 hours.
2018  */
2019 void
2021 {
2022  memset(&padding_current, 0, sizeof(padding_current));
2023 }
2024 
2025 /**
2026  * Copy our current cell counts into a structure for listing in our
2027  * extra-info descriptor. Also perform appropriate rounding and redaction.
2028  *
2029  * This function is called once every 24 hours.
2030  */
2031 #define MIN_CELL_COUNTS_TO_PUBLISH 1
2032 #define ROUND_CELL_COUNTS_TO 10000
2033 void
2034 rep_hist_prep_published_padding_counts(time_t now)
2035 {
2037 
2040  memset(&padding_published, 0, sizeof(padding_published));
2041  return;
2042  }
2043 
2045 #define ROUND_AND_SET_COUNT(x) (x) = round_uint64_to_next_multiple_of((x), \
2046  ROUND_CELL_COUNTS_TO)
2047  ROUND_AND_SET_COUNT(padding_published.read_pad_cell_count);
2048  ROUND_AND_SET_COUNT(padding_published.write_pad_cell_count);
2049  ROUND_AND_SET_COUNT(padding_published.read_drop_cell_count);
2050  ROUND_AND_SET_COUNT(padding_published.write_drop_cell_count);
2051  ROUND_AND_SET_COUNT(padding_published.write_cell_count);
2052  ROUND_AND_SET_COUNT(padding_published.read_cell_count);
2053  ROUND_AND_SET_COUNT(padding_published.enabled_read_cell_count);
2054  ROUND_AND_SET_COUNT(padding_published.enabled_read_pad_cell_count);
2055  ROUND_AND_SET_COUNT(padding_published.enabled_write_cell_count);
2056  ROUND_AND_SET_COUNT(padding_published.enabled_write_pad_cell_count);
2057 #undef ROUND_AND_SET_COUNT
2058 }
2059 
2060 /**
2061  * Returns an allocated string for extra-info documents for publishing
2062  * padding statistics from the last 24 hour interval.
2063  */
2064 char *
2066 {
2067  char *result = NULL;
2068 
2071  return NULL;
2072  }
2073 
2074  tor_asprintf(&result, "padding-counts %s (%d s)"
2075  " bin-size=%"PRIu64
2076  " write-drop=%"PRIu64
2077  " write-pad=%"PRIu64
2078  " write-total=%"PRIu64
2079  " read-drop=%"PRIu64
2080  " read-pad=%"PRIu64
2081  " read-total=%"PRIu64
2082  " enabled-read-pad=%"PRIu64
2083  " enabled-read-total=%"PRIu64
2084  " enabled-write-pad=%"PRIu64
2085  " enabled-write-total=%"PRIu64
2086  " max-chanpad-timers=%"PRIu64
2087  "\n",
2090  (uint64_t)ROUND_CELL_COUNTS_TO,
2102  );
2103 
2104  return result;
2105 }
2106 
2107 /** Log a heartbeat message explaining how many connections of each link
2108  * protocol version we have used.
2109  */
2110 void
2112 {
2113  smartlist_t *lines = smartlist_new();
2114 
2115  for (int i = 1; i <= MAX_LINK_PROTO; i++) {
2116  char *line = NULL;
2117  tor_asprintf(&line, "initiated %"PRIu64" and received "
2118  "%"PRIu64" v%d connections", link_proto_count[i][1],
2119  link_proto_count[i][0], i);
2120  smartlist_add(lines, line);
2121  }
2122 
2123  char *log_line = smartlist_join_strings(lines, "; ", 0, NULL);
2124 
2125  log_notice(LD_HEARTBEAT, "Since startup we %s.", log_line);
2126 
2127  SMARTLIST_FOREACH(lines, char *, s, tor_free(s));
2128  smartlist_free(lines);
2129  tor_free(log_line);
2130 }
2131 
2132 /** Free all storage held by the OR/link history caches, by the
2133  * bandwidth history arrays, by the port history, or by statistics . */
2134 void
2136 {
2137  hs_stats_free(hs_stats);
2138  digestmap_free(history_map, free_or_history);
2139 
2145 
2148  tor_free(s));
2149  smartlist_free(circuits_for_buffer_stats);
2151  }
2154 
2155  tor_assert_nonfatal(rephist_total_alloc == 0);
2156  tor_assert_nonfatal_once(rephist_total_num == 0);
2157 }
finish_writing_to_file
int finish_writing_to_file(open_file_t *file_data)
Definition: files.c:449
rep_hist_downrate_old_runs
time_t rep_hist_downrate_old_runs(time_t now)
Definition: rephist.c:369
padding_counts_t::first_published_at
char first_published_at[ISO_TIME_LEN+1]
Definition: rephist.c:165
tor_addr_compare
int tor_addr_compare(const tor_addr_t *addr1, const tor_addr_t *addr2, tor_addr_comparison_t how)
Definition: address.c:984
hs_stats
static hs_stats_t * hs_stats
Definition: rephist.c:1727
tor_free
#define tor_free(p)
Definition: malloc.h:52
or_history_t::weighted_run_length
unsigned long weighted_run_length
Definition: rephist.c:126
padding_counts_t::write_pad_cell_count
uint64_t write_pad_cell_count
Definition: rephist.c:149
networkstatus_t::valid_until
time_t valid_until
Definition: networkstatus_st.h:36
rep_hist_format_desc_stats
static char * rep_hist_format_desc_stats(time_t now)
Definition: rephist.c:1548
smartlist_split_string
int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep, int flags, int max)
Definition: smartlist_split.c:37
hex_str
const char * hex_str(const char *from, size_t fromlen)
Definition: binascii.c:34
rep_hist_reset_hs_stats
static void rep_hist_reset_hs_stats(time_t now)
Definition: rephist.c:1768
tor_addr_t
Definition: address.h:69
STABILITY_EPSILON
#define STABILITY_EPSILON
Definition: rephist.c:98
or_history_t::last_reached_addr
tor_addr_t last_reached_addr
Definition: rephist.c:118
stability_last_downrated
static time_t stability_last_downrated
Definition: rephist.c:178
rep_hist_get_uptime
long rep_hist_get_uptime(const char *id, time_t when)
Definition: rephist.c:477
MOCK_IMPL
#define MOCK_IMPL(rv, funcname, arglist)
Definition: testsupport.h:133
circuit_get_global_list
smartlist_t * circuit_get_global_list(void)
Definition: circuitlist.c:696
add_laplace_noise
int64_t add_laplace_noise(int64_t signal_, double random_, double delta_f, double epsilon)
Definition: laplace.c:51
rep_hist_load_mtbf_data
int rep_hist_load_mtbf_data(time_t now)
Definition: rephist.c:776
circ_buffer_stats_t::mean_num_cells_in_queue
double mean_num_cells_in_queue
Definition: rephist.c:1257
get_weighted_fractional_uptime
static double get_weighted_fractional_uptime(or_history_t *hist, time_t when)
Definition: rephist.c:450
tor_assert
#define tor_assert(expr)
Definition: util_bug.h:102
circ_buffer_stats_t::processed_cells
uint32_t processed_cells
Definition: rephist.c:1261
LD_BUG
#define LD_BUG
Definition: log.h:86
get_stability
static double get_stability(or_history_t *hist, time_t when)
Definition: rephist.c:414
hs_stats_free_
static void hs_stats_free_(hs_stats_t *victim_hs_stats)
Definition: rephist.c:1744
or_history_t
Definition: rephist.c:110
rep_hist_reset_exit_stats
void rep_hist_reset_exit_stats(time_t now)
Definition: rephist.c:1001
format_iso_time
void format_iso_time(char *buf, time_t t)
Definition: time_fmt.c:295
append_bytes_to_file
int append_bytes_to_file(const char *fname, const char *str, size_t len, int bin)
Definition: files.c:538
compare_int_
static int compare_int_(const void *x, const void *y)
Definition: rephist.c:1024
PADDING_TYPE_ENABLED_CELL
@ PADDING_TYPE_ENABLED_CELL
Definition: rephist.h:99
rep_hist_free_all
void rep_hist_free_all(void)
Definition: rephist.c:2135
base16_encode
void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:478
rep_hist_desc_stats_init
void rep_hist_desc_stats_init(time_t now)
Definition: rephist.c:1512
padding_counts_t
Definition: rephist.c:141
rep_hist_padding_count_timers
void rep_hist_padding_count_timers(uint64_t num_timers)
Definition: rephist.c:1953
EXIT_STATS_NUM_PORTS
#define EXIT_STATS_NUM_PORTS
Definition: rephist.c:971
rep_hist_exit_stats_term
void rep_hist_exit_stats_term(void)
Definition: rephist.c:1012
rep_hist_note_exit_stream_opened
void rep_hist_note_exit_stream_opened(uint16_t port)
Definition: rephist.c:1232
node_get_by_id
const node_t * node_get_by_id(const char *identity_digest)
Definition: nodelist.c:223
smartlist_add
void smartlist_add(smartlist_t *sl, void *element)
Definition: smartlist_core.c:117
get_total_weighted_time
static long get_total_weighted_time(or_history_t *hist, time_t when)
Definition: rephist.c:436
rep_hist_have_measured_enough_stability
int rep_hist_have_measured_enough_stability(void)
Definition: rephist.c:530
circ_buffer_stats_t::mean_time_cells_in_queue
double mean_time_cells_in_queue
Definition: rephist.c:1259
check_or_create_data_subdir
int check_or_create_data_subdir(const char *subdir)
Definition: config.c:6944
hs_stats_new
static hs_stats_t * hs_stats_new(void)
Definition: rephist.c:1731
rep_hist_exit_stats_init
void rep_hist_exit_stats_init(time_t now)
Definition: rephist.c:991
onion_handshakes_requested
STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1]
Definition: rephist.c:1653
rep_hist_note_desc_served
void rep_hist_note_desc_served(const char *desc)
Definition: rephist.c:1633
EXIT_STATS_ROUND_UP_BYTES
#define EXIT_STATS_ROUND_UP_BYTES
Definition: rephist.c:967
LD_HIST
#define LD_HIST
Definition: log.h:99
exit_streams
static uint32_t * exit_streams
Definition: rephist.c:984
smartlist_new
smartlist_t * smartlist_new(void)
Definition: smartlist_core.c:26
tor_addr_make_unspec
void tor_addr_make_unspec(tor_addr_t *a)
Definition: address.c:225
round_uint64_to_next_multiple_of
uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor)
Definition: muldiv.c:50
start_of_hs_stats_interval
static time_t start_of_hs_stats_interval
Definition: rephist.c:1713
CIRCUIT_IS_ORIGIN
#define CIRCUIT_IS_ORIGIN(c)
Definition: circuitlist.h:146
rep_hist_note_router_reachable
void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, const uint16_t at_port, time_t when)
Definition: rephist.c:228
served_descs
static digestmap_t * served_descs
Definition: rephist.c:1501
tor_parse_long
long tor_parse_long(const char *s, int base, long min, long max, int *ok, char **next)
Definition: parse_int.c:59
open_file_t
Definition: files.c:273
parse_iso_time
int parse_iso_time(const char *cp, time_t *t)
Definition: time_fmt.c:392
EXIT_STATS_ROUND_UP_STREAMS
#define EXIT_STATS_ROUND_UP_STREAMS
Definition: rephist.c:969
SMARTLIST_FOREACH
#define SMARTLIST_FOREACH(sl, type, var, cmd)
Definition: smartlist_foreach.h:112
rep_hist_buffer_stats_init
void rep_hist_buffer_stats_init(time_t now)
Definition: rephist.c:1248
networkstatus.h
Header file for networkstatus.c.
n_bogus_times
static int n_bogus_times
Definition: rephist.c:731
STABILITY_INTERVAL
#define STABILITY_INTERVAL
Definition: rephist.c:103
rep_hist_note_negotiated_link_proto
void rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
Definition: rephist.c:1937
rep_hist_buffer_stats_add_circ
void rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval)
Definition: rephist.c:1290
rep_history_clean
void rep_history_clean(time_t before)
Definition: rephist.c:580
rep_hist_padding_count_read
void rep_hist_padding_count_read(padding_type_t type)
Definition: rephist.c:1995
base16_decode
int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:506
hs_stats_t
Definition: rephist.c:1717
rep_hist_dump_stats
void rep_hist_dump_stats(time_t now, int severity)
Definition: rephist.c:541
REPHIST_CELL_PADDING_COUNTS_INTERVAL
#define REPHIST_CELL_PADDING_COUNTS_INTERVAL
Definition: rephist.h:103
padding_counts_t::maximum_chanpad_timers
uint64_t maximum_chanpad_timers
Definition: rephist.c:163
rep_hist_buffer_stats_term
void rep_hist_buffer_stats_term(void)
Definition: rephist.c:1340
circuitlist.h
Header file for circuitlist.c.
DIGEST_LEN
#define DIGEST_LEN
Definition: digest_sizes.h:20
get_or_history
static or_history_t * get_or_history(const char *id)
Definition: rephist.c:189
rep_hist_desc_stats_term
void rep_hist_desc_stats_term(void)
Definition: rephist.c:1535
rep_hist_add_buffer_stats
void rep_hist_add_buffer_stats(double mean_num_cells_in_queue, double mean_time_cells_in_queue, uint32_t processed_cells)
Definition: rephist.c:1271
rep_hist_reset_padding_counts
void rep_hist_reset_padding_counts(void)
Definition: rephist.c:2020
or_history_t::total_run_weights
double total_run_weights
Definition: rephist.c:131
rep_hist_reset_buffer_stats
void rep_hist_reset_buffer_stats(time_t now)
Definition: rephist.c:1348
rep_hist_get_weighted_time_known
long rep_hist_get_weighted_time_known(const char *id, time_t when)
Definition: rephist.c:518
PADDING_TYPE_ENABLED_TOTAL
@ PADDING_TYPE_ENABLED_TOTAL
Definition: rephist.h:97
padding_counts_t::read_pad_cell_count
uint64_t read_pad_cell_count
Definition: rephist.c:147
predicted_ports_free_all
void predicted_ports_free_all(void)
Definition: predict_ports.c:304
rep_hist_log_link_protocol_counts
void rep_hist_log_link_protocol_counts(void)
Definition: rephist.c:2111
hs_stats_t::rp_relay_cells_seen
uint64_t rp_relay_cells_seen
Definition: rephist.c:1719
networkstatus_get_latest_consensus
networkstatus_t * networkstatus_get_latest_consensus(void)
Definition: networkstatus.c:1390
padding_counts_t::enabled_write_pad_cell_count
uint64_t enabled_write_pad_cell_count
Definition: rephist.c:157
node_t
Definition: node_st.h:34
rep_hist_init
void rep_hist_init(void)
Definition: rephist.c:220
RFTS_IGNORE_MISSING
#define RFTS_IGNORE_MISSING
Definition: files.h:97
hs_stats_t::onions_seen_this_period
digestmap_t * onions_seen_this_period
Definition: rephist.c:1723
rep_hist_get_stability
double rep_hist_get_stability(const char *id, time_t when)
Definition: rephist.c:490
padding_type_t
padding_type_t
Definition: rephist.h:89
rephist_total_num
uint32_t rephist_total_num
Definition: rephist.c:94
rep_hist_hs_stats_term
void rep_hist_hs_stats_term(void)
Definition: rephist.c:1785
crypto_pk_get_digest
int crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out)
Definition: crypto_rsa.c:356
rep_hist_hs_stats_init
void rep_hist_hs_stats_init(time_t now)
Definition: rephist.c:1756
rep_hist_format_hs_stats
static char * rep_hist_format_hs_stats(time_t now)
Definition: rephist.c:1856
rep_hist_hs_stats_write
time_t rep_hist_hs_stats_write(time_t now)
Definition: rephist.c:1902
buffer_stats_compare_entries_
static int buffer_stats_compare_entries_(const void **_a, const void **_b)
Definition: rephist.c:1326
tor_digest_is_zero
int tor_digest_is_zero(const char *digest)
Definition: util_string.c:96
exit_bytes_read
static uint64_t * exit_bytes_read
Definition: rephist.c:980
padding_counts_t::write_drop_cell_count
uint64_t write_drop_cell_count
Definition: rephist.c:161
escaped
const char * escaped(const char *s)
Definition: escape.c:126
strcmpstart
int strcmpstart(const char *s1, const char *s2)
Definition: util_string.c:206
find_next_with
static int find_next_with(smartlist_t *sl, int i, const char *prefix)
Definition: rephist.c:718
authmode.h
Header file for directory authority mode.
circuits_for_buffer_stats
static smartlist_t * circuits_for_buffer_stats
Definition: rephist.c:1265
circuit_t
Definition: circuit_st.h:61
nodelist.h
Header file for nodelist.c.
LD_HEARTBEAT
#define LD_HEARTBEAT
Definition: log.h:103
rep_hist_get_weighted_fractional_uptime
double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when)
Definition: rephist.c:502
laplace.h
Header for laplace.c.
rep_hist_format_buffer_stats
char * rep_hist_format_buffer_stats(time_t now)
Definition: rephist.c:1362
predict_ports.h
Header file for predict_ports.c.
rep_hist_get_circuit_handshake_assigned
int rep_hist_get_circuit_handshake_assigned(uint16_t type)
Definition: rephist.c:1686
conn_stats_free_all
void conn_stats_free_all(void)
Definition: connstats.c:112
circuit_t::timestamp_created
struct timeval timestamp_created
Definition: circuit_st.h:168
abort_writing_to_file
int abort_writing_to_file(open_file_t *file_data)
Definition: files.c:457
EXIT_STATS_TOP_N_PORTS
#define EXIT_STATS_TOP_N_PORTS
Definition: rephist.c:973
padding_counts_t::read_drop_cell_count
uint64_t read_drop_cell_count
Definition: rephist.c:159
round_uint32_to_next_multiple_of
uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor)
Definition: muldiv.c:35
tor_sscanf
int tor_sscanf(const char *buf, const char *pattern,...)
Definition: scanf.c:309
PADDING_TYPE_TOTAL
@ PADDING_TYPE_TOTAL
Definition: rephist.h:95
rep_hist_buffer_stats_write
time_t rep_hist_buffer_stats_write(time_t now)
Definition: rephist.c:1465
STABILITY_ALPHA
#define STABILITY_ALPHA
Definition: rephist.c:101
rep_hist_padding_count_write
void rep_hist_padding_count_write(padding_type_t type)
Definition: rephist.c:1967
parse_possibly_bad_iso_time
static int parse_possibly_bad_iso_time(const char *s, time_t *time_out)
Definition: rephist.c:735
rep_hist_get_circuit_handshake_requested
int rep_hist_get_circuit_handshake_requested(uint16_t type)
Definition: rephist.c:1676
crypto_rand.h
Common functions for using (pseudo-)random number generators.
get_options
const or_options_t * get_options(void)
Definition: config.c:928
rep_hist_note_circuit_handshake_requested
void rep_hist_note_circuit_handshake_requested(uint16_t type)
Definition: rephist.c:1659
padding_current
static padding_counts_t padding_current
Definition: rephist.c:170
rep_hist_note_router_unreachable
void rep_hist_note_router_unreachable(const char *id, time_t when)
Definition: rephist.c:301
rep_hist_make_router_pessimal
void rep_hist_make_router_pessimal(const char *id, time_t when)
Definition: rephist.c:355
write_to_data_subdir
int write_to_data_subdir(const char *subdir, const char *fname, const char *str, const char *descr)
Definition: config.c:6963
padding_counts_t::enabled_write_cell_count
uint64_t enabled_write_cell_count
Definition: rephist.c:153
smartlist_clear
void smartlist_clear(smartlist_t *sl)
Definition: smartlist_core.c:50
padding_counts_t::read_cell_count
uint64_t read_cell_count
Definition: rephist.c:143
rep_hist_stored_maybe_new_hs
void rep_hist_stored_maybe_new_hs(const crypto_pk_t *pubkey)
Definition: rephist.c:1805
total_descriptor_downloads
static unsigned long total_descriptor_downloads
Definition: rephist.c:1505
circ_buffer_stats_t
Definition: rephist.c:1255
rep_hist_format_exit_stats
char * rep_hist_format_exit_stats(time_t now)
Definition: rephist.c:1033
rep_hist_note_circuit_handshake_assigned
void rep_hist_note_circuit_handshake_assigned(uint16_t type)
Definition: rephist.c:1668
connstats.h
Header for feature/stats/connstats.c.
or_history_t::changed
time_t changed
Definition: rephist.c:114
free_or_history
static void free_or_history(void *_hist)
Definition: rephist.c:210
networkstatus_t::valid_after
time_t valid_after
Definition: networkstatus_st.h:33
rep_hist_reset_desc_stats
static void rep_hist_reset_desc_stats(time_t now)
Definition: rephist.c:1526
tor_addr_is_null
int tor_addr_is_null(const tor_addr_t *addr)
Definition: address.c:780
exit_bytes_written
static uint64_t * exit_bytes_written
Definition: rephist.c:982
tor_asprintf
int tor_asprintf(char **strp, const char *fmt,...)
Definition: printf.c:75
crypto_rand_double
double crypto_rand_double(void)
Definition: crypto_rand_numeric.c:126
SMARTLIST_FOREACH_BEGIN
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
Definition: smartlist_foreach.h:78
node_get_nickname
const char * node_get_nickname(const node_t *node)
Definition: nodelist.c:1369
crypto_pk_t
Definition: crypto_rsa_nss.c:37
or_options_t::TestingTorNetwork
int TestingTorNetwork
Definition: or_options_st.h:791
padding_published
static padding_counts_t padding_published
Definition: rephist.c:174
or_circuit_t::processed_cells
uint32_t processed_cells
Definition: or_circuit_st.h:68
start_of_buffer_stats_interval
static time_t start_of_buffer_stats_interval
Definition: rephist.c:1244
HEX_DIGEST_LEN
#define HEX_DIGEST_LEN
Definition: crypto_digest.h:35
padding_counts_t::enabled_read_pad_cell_count
uint64_t enabled_read_pad_cell_count
Definition: rephist.c:155
rep_hist_note_exit_bytes
void rep_hist_note_exit_bytes(uint16_t port, size_t num_written, size_t num_read)
Definition: rephist.c:1218
or_history_t::since
time_t since
Definition: rephist.c:112
rep_hist_desc_stats_write
time_t rep_hist_desc_stats_write(time_t now)
Definition: rephist.c:1602
connection_or_digest_is_known_relay
int connection_or_digest_is_known_relay(const char *id_digest)
Definition: connection_or.c:786
order.h
Header for order.c.
format_local_iso_time
void format_local_iso_time(char *buf, time_t t)
Definition: time_fmt.c:285
rep_hist_seen_new_rp_cell
void rep_hist_seen_new_rp_cell(void)
Definition: rephist.c:1792
or_circuit_t
Definition: or_circuit_st.h:21
correct_time
static time_t correct_time(time_t t, time_t now, time_t stored_at, time_t started_measuring)
Definition: rephist.c:756
config.h
Header file for config.c.
or_history_t::start_of_run
time_t start_of_run
Definition: rephist.c:129
PADDING_TYPE_DROP
@ PADDING_TYPE_DROP
Definition: rephist.h:91
smartlist_add_asprintf
void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern,...)
Definition: smartlist.c:36
tor_log
void tor_log(int severity, log_domain_mask_t domain, const char *format,...)
Definition: log.c:628
rephist_total_alloc
uint64_t rephist_total_alloc
Definition: rephist.c:92
or_circuit_t::total_cell_waiting_time
uint64_t total_cell_waiting_time
Definition: or_circuit_st.h:73
start_of_served_descs_stats_interval
static time_t start_of_served_descs_stats_interval
Definition: rephist.c:1508
rep_hist_get_padding_count_lines
char * rep_hist_get_padding_count_lines(void)
Definition: rephist.c:2065
STATIC
#define STATIC
Definition: testsupport.h:32
networkstatus_t::fresh_until
time_t fresh_until
Definition: networkstatus_st.h:34
rep_hist_exit_stats_write
time_t rep_hist_exit_stats_write(time_t now)
Definition: rephist.c:1188
MIN_CELL_COUNTS_TO_PUBLISH
#define MIN_CELL_COUNTS_TO_PUBLISH
Definition: rephist.c:2031
networkstatus_st.h
Networkstatus consensus/vote structure.
start_writing_to_stdio_file
FILE * start_writing_to_stdio_file(const char *fname, int open_flags, int mode, open_file_t **data_out)
Definition: files.c:382
smartlist_sort
void smartlist_sort(smartlist_t *sl, int(*compare)(const void **a, const void **b))
Definition: smartlist.c:334
TO_OR_CIRCUIT
or_circuit_t * TO_OR_CIRCUIT(circuit_t *x)
Definition: circuitlist.c:156
networkstatus_t
Definition: networkstatus_st.h:26
padding_counts_t::write_cell_count
uint64_t write_cell_count
Definition: rephist.c:145
start_of_exit_stats_interval
static time_t start_of_exit_stats_interval
Definition: rephist.c:987
PADDING_TYPE_CELL
@ PADDING_TYPE_CELL
Definition: rephist.h:93
smartlist_t
Definition: smartlist_core.h:26
smartlist_join_strings
char * smartlist_join_strings(smartlist_t *sl, const char *join, int terminate, size_t *len_out)
Definition: smartlist.c:279
padding_counts_t::enabled_read_cell_count
uint64_t enabled_read_cell_count
Definition: rephist.c:151
rephist.h
Header file for rephist.c.
or_history_t::last_reached_port
uint16_t last_reached_port
Definition: rephist.c:121
connection_or.h
Header file for connection_or.c.
authdir_mode
int authdir_mode(const or_options_t *options)
Definition: authmode.c:25
tor_addr_copy
void tor_addr_copy(tor_addr_t *dest, const tor_addr_t *src)
Definition: address.c:933
history_map
static digestmap_t * history_map
Definition: rephist.c:184
rep_hist_log_circuit_handshake_stats
void rep_hist_log_circuit_handshake_stats(time_t now)
Definition: rephist.c:1696
or.h
Master header file for Tor-specific functionality.
rep_hist_record_mtbf_data
int rep_hist_record_mtbf_data(time_t now, int missing_means_down)
Definition: rephist.c:613