Tor  0.4.7.0-alpha-dev
geoip_stats.c
1 /* Copyright (c) 2007-2021, The Tor Project, Inc. */
2 /* See LICENSE for licensing information */
3 
4 /**
5  * \file geoip.c
6  * \brief Functions related to maintaining an IP-to-country database;
7  * to summarizing client connections by country to entry guards, bridges,
8  * and directory servers; and for statistics on answering network status
9  * requests.
10  *
11  * There are two main kinds of functions in this module: geoip functions,
12  * which map groups of IPv4 and IPv6 addresses to country codes, and
13  * statistical functions, which collect statistics about different kinds of
14  * per-country usage.
15  *
16  * The geoip lookup tables are implemented as sorted lists of disjoint address
17  * ranges, each mapping to a singleton geoip_country_t. These country objects
18  * are also indexed by their names in a hashtable.
19  *
20  * The tables are populated from disk at startup by the geoip_load_file()
21  * function. For more information on the file format they read, see that
22  * function. See the scripts and the README file in src/config for more
23  * information about how those files are generated.
24  *
25  * Tor uses GeoIP information in order to implement user requests (such as
26  * ExcludeNodes {cc}), and to keep track of how much usage relays are getting
27  * for each country.
28  */
29 
30 #include "core/or/or.h"
31 
32 #include "ht.h"
33 #include "lib/buf/buffers.h"
34 #include "app/config/config.h"
36 #include "feature/client/dnsserv.h"
37 #include "core/or/dos.h"
38 #include "lib/geoip/geoip.h"
41 
42 #include "lib/container/order.h"
43 #include "lib/time/tvdiff.h"
44 
45 /** Number of entries in n_v3_ns_requests */
46 static size_t n_v3_ns_requests_len = 0;
47 /** Array, indexed by country index, of number of v3 networkstatus requests
48  * received from that country */
49 static uint32_t *n_v3_ns_requests;
50 
51 /* Total size in bytes of the geoip client history cache. Used by the OOM
52  * handler. */
53 static size_t geoip_client_history_cache_size;
54 
55 /* Increment the geoip client history cache size counter with the given bytes.
56  * This prevents an overflow and set it to its maximum in that case. */
57 static inline void
58 geoip_increment_client_history_cache_size(size_t bytes)
59 {
60  /* This is shockingly high, lets log it so it can be reported. */
61  IF_BUG_ONCE(geoip_client_history_cache_size > (SIZE_MAX - bytes)) {
62  geoip_client_history_cache_size = SIZE_MAX;
63  return;
64  }
65  geoip_client_history_cache_size += bytes;
66 }
67 
68 /* Decrement the geoip client history cache size counter with the given bytes.
69  * This prevents an underflow and set it to 0 in that case. */
70 static inline void
71 geoip_decrement_client_history_cache_size(size_t bytes)
72 {
73  /* Going below 0 means that we either allocated an entry without
74  * incrementing the counter or we have different sizes when allocating and
75  * freeing. It shouldn't happened so log it. */
76  IF_BUG_ONCE(geoip_client_history_cache_size < bytes) {
77  geoip_client_history_cache_size = 0;
78  return;
79  }
80  geoip_client_history_cache_size -= bytes;
81 }
82 
83 /** Add 1 to the count of v3 ns requests received from <b>country</b>. */
84 static void
85 increment_v3_ns_request(country_t country)
86 {
87  if (country < 0)
88  return;
89 
90  if ((size_t)country >= n_v3_ns_requests_len) {
91  /* We need to reallocate the array. */
92  size_t new_len;
93  if (n_v3_ns_requests_len == 0)
94  new_len = 256;
95  else
96  new_len = n_v3_ns_requests_len * 2;
97  if (new_len <= (size_t)country)
98  new_len = ((size_t)country)+1;
99  n_v3_ns_requests = tor_reallocarray(n_v3_ns_requests, new_len,
100  sizeof(uint32_t));
101  memset(n_v3_ns_requests + n_v3_ns_requests_len, 0,
102  sizeof(uint32_t)*(new_len - n_v3_ns_requests_len));
103  n_v3_ns_requests_len = new_len;
104  }
105 
106  n_v3_ns_requests[country] += 1;
107 }
108 
109 /** Return 1 if we should collect geoip stats on bridge users, and
110  * include them in our extrainfo descriptor. Else return 0. */
111 int
113 {
114  return options->BridgeRelay && options->BridgeRecordUsageByCountry;
115 }
116 
117 /** Largest allowable value for last_seen_in_minutes. (It's a 30-bit field,
118  * so it can hold up to (1u<<30)-1, or 0x3fffffffu.
119  */
120 #define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu
121 
122 /** Map from client IP address to last time seen. */
123 static HT_HEAD(clientmap, clientmap_entry_t) client_history =
124  HT_INITIALIZER();
125 
126 /** Hashtable helper: compute a hash of a clientmap_entry_t. */
127 static inline unsigned
128 clientmap_entry_hash(const clientmap_entry_t *a)
129 {
130  unsigned h = (unsigned) tor_addr_hash(&a->addr);
131 
132  if (a->transport_name)
133  h += (unsigned) siphash24g(a->transport_name, strlen(a->transport_name));
134 
135  return h;
136 }
137 /** Hashtable helper: compare two clientmap_entry_t values for equality. */
138 static inline int
139 clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b)
140 {
141  if (strcmp_opt(a->transport_name, b->transport_name))
142  return 0;
143 
144  return !tor_addr_compare(&a->addr, &b->addr, CMP_EXACT) &&
145  a->action == b->action;
146 }
147 
148 HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
149  clientmap_entries_eq);
150 HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
151  clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_);
152 
153 #define clientmap_entry_free(ent) \
154  FREE_AND_NULL(clientmap_entry_t, clientmap_entry_free_, ent)
155 
156 /** Return the size of a client map entry. */
157 static inline size_t
158 clientmap_entry_size(const clientmap_entry_t *ent)
159 {
160  tor_assert(ent);
161  return (sizeof(clientmap_entry_t) +
162  (ent->transport_name ? strlen(ent->transport_name) : 0));
163 }
164 
165 /** Free all storage held by <b>ent</b>. */
166 static void
167 clientmap_entry_free_(clientmap_entry_t *ent)
168 {
169  if (!ent)
170  return;
171 
172  /* This entry is about to be freed so pass it to the DoS subsystem to see if
173  * any actions can be taken about it. */
174  dos_geoip_entry_about_to_free(ent);
175  geoip_decrement_client_history_cache_size(clientmap_entry_size(ent));
176 
177  tor_free(ent->transport_name);
178  tor_free(ent);
179 }
180 
181 /* Return a newly allocated clientmap entry with the given action and address
182  * that are mandatory. The transport_name can be optional. This can't fail. */
183 static clientmap_entry_t *
184 clientmap_entry_new(geoip_client_action_t action, const tor_addr_t *addr,
185  const char *transport_name)
186 {
187  clientmap_entry_t *entry;
188 
189  tor_assert(action == GEOIP_CLIENT_CONNECT ||
190  action == GEOIP_CLIENT_NETWORKSTATUS);
191  tor_assert(addr);
192 
193  entry = tor_malloc_zero(sizeof(clientmap_entry_t));
194  entry->action = action;
195  tor_addr_copy(&entry->addr, addr);
196  if (transport_name) {
197  entry->transport_name = tor_strdup(transport_name);
198  }
199  /* Initialize the DoS object. */
200  dos_geoip_entry_init(entry);
201 
202  /* Allocated and initialized, note down its size for the OOM handler. */
203  geoip_increment_client_history_cache_size(clientmap_entry_size(entry));
204 
205  return entry;
206 }
207 
208 /** Clear history of connecting clients used by entry and bridge stats. */
209 static void
210 client_history_clear(void)
211 {
212  clientmap_entry_t **ent, **next, *this;
213  for (ent = HT_START(clientmap, &client_history); ent != NULL;
214  ent = next) {
215  if ((*ent)->action == GEOIP_CLIENT_CONNECT) {
216  this = *ent;
217  next = HT_NEXT_RMV(clientmap, &client_history, ent);
218  clientmap_entry_free(this);
219  } else {
220  next = HT_NEXT(clientmap, &client_history, ent);
221  }
222  }
223 }
224 
225 /** Note that we've seen a client connect from the IP <b>addr</b>
226  * at time <b>now</b>. Ignored by all but bridges and directories if
227  * configured accordingly. */
228 void
230  const tor_addr_t *addr,
231  const char *transport_name,
232  time_t now)
233 {
234  const or_options_t *options = get_options();
235  clientmap_entry_t *ent;
236 
237  if (action == GEOIP_CLIENT_CONNECT) {
238  /* Only remember statistics if the DoS mitigation subsystem is enabled. If
239  * not, only if as entry guard or as bridge. */
240  if (!dos_enabled()) {
241  if (!options->EntryStatistics && !should_record_bridge_info(options)) {
242  return;
243  }
244  }
245  } else {
246  /* Only gather directory-request statistics if configured, and
247  * forcibly disable them on bridge authorities. */
248  if (!options->DirReqStatistics || options->BridgeAuthoritativeDir)
249  return;
250  }
251 
252  log_debug(LD_GENERAL, "Seen client from '%s' with transport '%s'.",
253  safe_str_client(fmt_addr((addr))),
254  transport_name ? transport_name : "<no transport>");
255 
256  ent = geoip_lookup_client(addr, transport_name, action);
257  if (! ent) {
258  ent = clientmap_entry_new(action, addr, transport_name);
259  HT_INSERT(clientmap, &client_history, ent);
260  }
261  if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0)
262  ent->last_seen_in_minutes = (unsigned)(now/60);
263  else
264  ent->last_seen_in_minutes = 0;
265 
266  if (action == GEOIP_CLIENT_NETWORKSTATUS) {
267  int country_idx = geoip_get_country_by_addr(addr);
268  if (country_idx < 0)
269  country_idx = 0; /** unresolved requests are stored at index 0. */
270  IF_BUG_ONCE(country_idx > COUNTRY_MAX) {
271  return;
272  }
273  increment_v3_ns_request((country_t) country_idx);
274  }
275 }
276 
277 /** HT_FOREACH helper: remove a clientmap_entry_t from the hashtable if it's
278  * older than a certain time. */
279 static int
280 remove_old_client_helper_(struct clientmap_entry_t *ent, void *_cutoff)
281 {
282  time_t cutoff = *(time_t*)_cutoff / 60;
283  if (ent->last_seen_in_minutes < cutoff) {
284  clientmap_entry_free(ent);
285  return 1;
286  } else {
287  return 0;
288  }
289 }
290 
291 /** Forget about all clients that haven't connected since <b>cutoff</b>. */
292 void
294 {
295  clientmap_HT_FOREACH_FN(&client_history,
296  remove_old_client_helper_,
297  &cutoff);
298 }
299 
300 /* Return a client entry object matching the given address, transport name and
301  * geoip action from the clientmap. NULL if not found. The transport_name can
302  * be NULL. */
304 geoip_lookup_client(const tor_addr_t *addr, const char *transport_name,
305  geoip_client_action_t action)
306 {
307  clientmap_entry_t lookup;
308 
309  tor_assert(addr);
310 
311  /* We always look for a client connection with no transport. */
312  tor_addr_copy(&lookup.addr, addr);
313  lookup.action = action;
314  lookup.transport_name = (char *) transport_name;
315 
316  return HT_FIND(clientmap, &client_history, &lookup);
317 }
318 
319 /* Cleanup client entries older than the cutoff. Used for the OOM. Return the
320  * number of bytes freed. If 0 is returned, nothing was freed. */
321 static size_t
322 oom_clean_client_entries(time_t cutoff)
323 {
324  size_t bytes = 0;
325  clientmap_entry_t **ent, **ent_next;
326 
327  for (ent = HT_START(clientmap, &client_history); ent; ent = ent_next) {
328  clientmap_entry_t *entry = *ent;
329  if (entry->last_seen_in_minutes < (cutoff / 60)) {
330  ent_next = HT_NEXT_RMV(clientmap, &client_history, ent);
331  bytes += clientmap_entry_size(entry);
332  clientmap_entry_free(entry);
333  } else {
334  ent_next = HT_NEXT(clientmap, &client_history, ent);
335  }
336  }
337  return bytes;
338 }
339 
340 /* Below this minimum lifetime, the OOM won't cleanup any entries. */
341 #define GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF (4 * 60 * 60)
342 /* The OOM moves the cutoff by that much every run. */
343 #define GEOIP_CLIENT_CACHE_OOM_STEP (15 * 50)
344 
345 /* Cleanup the geoip client history cache called from the OOM handler. Return
346  * the amount of bytes removed. This can return a value below or above
347  * min_remove_bytes but will stop as oon as the min_remove_bytes has been
348  * reached. */
349 size_t
350 geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes)
351 {
352  time_t k;
353  size_t bytes_removed = 0;
354 
355  /* Our OOM handler called with 0 bytes to remove is a code flow error. */
356  tor_assert(min_remove_bytes != 0);
357 
358  /* Set k to the initial cutoff of an entry. We then going to move it by step
359  * to try to remove as much as we can. */
360  k = WRITE_STATS_INTERVAL;
361 
362  do {
363  time_t cutoff;
364 
365  /* If k has reached the minimum lifetime, we have to stop else we might
366  * remove every single entries which would be pretty bad for the DoS
367  * mitigation subsystem if by just filling the geoip cache, it was enough
368  * to trigger the OOM and clean every single entries. */
369  if (k <= GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF) {
370  break;
371  }
372 
373  cutoff = now - k;
374  bytes_removed += oom_clean_client_entries(cutoff);
375  k -= GEOIP_CLIENT_CACHE_OOM_STEP;
376  } while (bytes_removed < min_remove_bytes);
377 
378  return bytes_removed;
379 }
380 
381 /* Return the total size in bytes of the client history cache. */
382 size_t
383 geoip_client_cache_total_allocation(void)
384 {
385  return geoip_client_history_cache_size;
386 }
387 
388 /** How many responses are we giving to clients requesting v3 network
389  * statuses? */
390 static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM];
391 
392 /** Note that we've rejected a client's request for a v3 network status
393  * for reason <b>reason</b> at time <b>now</b>. */
394 void
396 {
397  static int arrays_initialized = 0;
398  if (!get_options()->DirReqStatistics)
399  return;
400  if (!arrays_initialized) {
401  memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
402  arrays_initialized = 1;
403  }
404  tor_assert(response < GEOIP_NS_RESPONSE_NUM);
405  ns_v3_responses[response]++;
406 }
407 
408 /** Do not mention any country from which fewer than this number of IPs have
409  * connected. This conceivably avoids reporting information that could
410  * deanonymize users, though analysis is lacking. */
411 #define MIN_IPS_TO_NOTE_COUNTRY 1
412 /** Do not report any geoip data at all if we have fewer than this number of
413  * IPs to report about. */
414 #define MIN_IPS_TO_NOTE_ANYTHING 1
415 /** When reporting geoip data about countries, round up to the nearest
416  * multiple of this value. */
417 #define IP_GRANULARITY 8
418 
419 /** Helper type: used to sort per-country totals by value. */
420 typedef struct c_hist_t {
421  char country[3]; /**< Two-letter country code. */
422  unsigned total; /**< Total IP addresses seen in this country. */
423 } c_hist_t;
424 
425 /** Sorting helper: return -1, 1, or 0 based on comparison of two
426  * geoip_ipv4_entry_t. Sort in descending order of total, and then by country
427  * code. */
428 static int
429 c_hist_compare_(const void **_a, const void **_b)
430 {
431  const c_hist_t *a = *_a, *b = *_b;
432  if (a->total > b->total)
433  return -1;
434  else if (a->total < b->total)
435  return 1;
436  else
437  return strcmp(a->country, b->country);
438 }
439 
440 /** When there are incomplete directory requests at the end of a 24-hour
441  * period, consider those requests running for longer than this timeout as
442  * failed, the others as still running. */
443 #define DIRREQ_TIMEOUT (10*60)
444 
445 /** Entry in a map from either chan->global_identifier for direct requests
446  * or a unique circuit identifier for tunneled requests to request time,
447  * response size, and completion time of a network status request. Used to
448  * measure download times of requests to derive average client
449  * bandwidths. */
450 typedef struct dirreq_map_entry_t {
451  HT_ENTRY(dirreq_map_entry_t) node;
452  /** Unique identifier for this network status request; this is either the
453  * chan->global_identifier of the dir channel (direct request) or a new
454  * locally unique identifier of a circuit (tunneled request). This ID is
455  * only unique among other direct or tunneled requests, respectively. */
456  uint64_t dirreq_id;
457  unsigned int state:3; /**< State of this directory request. */
458  unsigned int type:1; /**< Is this a direct or a tunneled request? */
459  unsigned int completed:1; /**< Is this request complete? */
460  /** When did we receive the request and started sending the response? */
461  struct timeval request_time;
462  size_t response_size; /**< What is the size of the response in bytes? */
463  struct timeval completion_time; /**< When did the request succeed? */
465 
466 /** Map of all directory requests asking for v2 or v3 network statuses in
467  * the current geoip-stats interval. Values are
468  * of type *<b>dirreq_map_entry_t</b>. */
469 static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map =
470  HT_INITIALIZER();
471 
472 static int
473 dirreq_map_ent_eq(const dirreq_map_entry_t *a,
474  const dirreq_map_entry_t *b)
475 {
476  return a->dirreq_id == b->dirreq_id && a->type == b->type;
477 }
478 
479 /* DOCDOC dirreq_map_ent_hash */
480 static unsigned
481 dirreq_map_ent_hash(const dirreq_map_entry_t *entry)
482 {
483  unsigned u = (unsigned) entry->dirreq_id;
484  u += entry->type << 20;
485  return u;
486 }
487 
488 HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
489  dirreq_map_ent_eq);
490 HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
491  dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_);
492 
493 /** Helper: Put <b>entry</b> into map of directory requests using
494  * <b>type</b> and <b>dirreq_id</b> as key parts. If there is
495  * already an entry for that key, print out a BUG warning and return. */
496 static void
497 dirreq_map_put_(dirreq_map_entry_t *entry, dirreq_type_t type,
498  uint64_t dirreq_id)
499 {
500  dirreq_map_entry_t *old_ent;
501  tor_assert(entry->type == type);
502  tor_assert(entry->dirreq_id == dirreq_id);
503 
504  /* XXXX we could switch this to HT_INSERT some time, since it seems that
505  * this bug doesn't happen. But since this function doesn't seem to be
506  * critical-path, it's sane to leave it alone. */
507  old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry);
508  if (old_ent && old_ent != entry) {
509  log_warn(LD_BUG, "Error when putting directory request into local "
510  "map. There was already an entry for the same identifier.");
511  return;
512  }
513 }
514 
515 /** Helper: Look up and return an entry in the map of directory requests
516  * using <b>type</b> and <b>dirreq_id</b> as key parts. If there
517  * is no such entry, return NULL. */
518 static dirreq_map_entry_t *
519 dirreq_map_get_(dirreq_type_t type, uint64_t dirreq_id)
520 {
521  dirreq_map_entry_t lookup;
522  lookup.type = type;
523  lookup.dirreq_id = dirreq_id;
524  return HT_FIND(dirreqmap, &dirreq_map, &lookup);
525 }
526 
527 /** Note that an either direct or tunneled (see <b>type</b>) directory
528  * request for a v3 network status with unique ID <b>dirreq_id</b> of size
529  * <b>response_size</b> has started. */
530 void
531 geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
532  dirreq_type_t type)
533 {
534  dirreq_map_entry_t *ent;
535  if (!get_options()->DirReqStatistics)
536  return;
537  ent = tor_malloc_zero(sizeof(dirreq_map_entry_t));
538  ent->dirreq_id = dirreq_id;
539  tor_gettimeofday(&ent->request_time);
540  ent->response_size = response_size;
541  ent->type = type;
542  dirreq_map_put_(ent, type, dirreq_id);
543 }
544 
545 /** Change the state of the either direct or tunneled (see <b>type</b>)
546  * directory request with <b>dirreq_id</b> to <b>new_state</b> and
547  * possibly mark it as completed. If no entry can be found for the given
548  * key parts (e.g., if this is a directory request that we are not
549  * measuring, or one that was started in the previous measurement period),
550  * or if the state cannot be advanced to <b>new_state</b>, do nothing. */
551 void
552 geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
553  dirreq_state_t new_state)
554 {
555  dirreq_map_entry_t *ent;
556  if (!get_options()->DirReqStatistics)
557  return;
558  ent = dirreq_map_get_(type, dirreq_id);
559  if (!ent)
560  return;
561  if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS)
562  return;
563  if (new_state - 1 != ent->state)
564  return;
565  ent->state = new_state;
566  if ((type == DIRREQ_DIRECT &&
567  new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) ||
568  (type == DIRREQ_TUNNELED &&
569  new_state == DIRREQ_CHANNEL_BUFFER_FLUSHED)) {
570  tor_gettimeofday(&ent->completion_time);
571  ent->completed = 1;
572  }
573 }
574 
575 /** Return the bridge-ip-transports string that should be inserted in
576  * our extra-info descriptor. Return NULL if the bridge-ip-transports
577  * line should be empty. */
578 char *
580 {
581  unsigned granularity = IP_GRANULARITY;
582  /** String hash table (name of transport) -> (number of users). */
583  strmap_t *transport_counts = strmap_new();
584 
585  /** Smartlist that contains copies of the names of the transports
586  that have been used. */
587  smartlist_t *transports_used = smartlist_new();
588 
589  /* Special string to signify that no transport was used for this
590  connection. Pluggable transport names can't have symbols in their
591  names, so this string will never collide with a real transport. */
592  static const char* no_transport_str = "<OR>";
593 
594  clientmap_entry_t **ent;
595  smartlist_t *string_chunks = smartlist_new();
596  char *the_string = NULL;
597 
598  /* If we haven't seen any clients yet, return NULL. */
599  if (HT_EMPTY(&client_history))
600  goto done;
601 
602  /** We do the following steps to form the transport history string:
603  * a) Foreach client that uses a pluggable transport, we increase the
604  * times that transport was used by one. If the client did not use
605  * a transport, we increase the number of times someone connected
606  * without obfuscation.
607  * b) Foreach transport we observed, we write its transport history
608  * string and push it to string_chunks. So, for example, if we've
609  * seen 665 obfs2 clients, we write "obfs2=665".
610  * c) We concatenate string_chunks to form the final string.
611  */
612 
613  log_debug(LD_GENERAL,"Starting iteration for transport history. %d clients.",
614  HT_SIZE(&client_history));
615 
616  /* Loop through all clients. */
617  HT_FOREACH(ent, clientmap, &client_history) {
618  uintptr_t val;
619  void *ptr;
620  const char *transport_name = (*ent)->transport_name;
621  if (!transport_name)
622  transport_name = no_transport_str;
623 
624  /* Increase the count for this transport name. */
625  ptr = strmap_get(transport_counts, transport_name);
626  val = (uintptr_t)ptr;
627  val++;
628  ptr = (void*)val;
629  strmap_set(transport_counts, transport_name, ptr);
630 
631  /* If it's the first time we see this transport, note it. */
632  if (val == 1)
633  smartlist_add_strdup(transports_used, transport_name);
634 
635  log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. "
636  "I've now seen %d clients.",
637  safe_str_client(fmt_addr(&(*ent)->addr)),
638  transport_name ? transport_name : "<no transport>",
639  (int)val);
640  }
641 
642  /* Sort the transport names (helps with unit testing). */
643  smartlist_sort_strings(transports_used);
644 
645  /* Loop through all seen transports. */
646  SMARTLIST_FOREACH_BEGIN(transports_used, const char *, transport_name) {
647  void *transport_count_ptr = strmap_get(transport_counts, transport_name);
648  uintptr_t transport_count = (uintptr_t) transport_count_ptr;
649 
650  log_debug(LD_GENERAL, "We got %"PRIu64" clients with transport '%s'.",
651  ((uint64_t)transport_count), transport_name);
652 
653  smartlist_add_asprintf(string_chunks, "%s=%"PRIu64,
654  transport_name,
656  (uint64_t)transport_count,
657  granularity)));
658  } SMARTLIST_FOREACH_END(transport_name);
659 
660  the_string = smartlist_join_strings(string_chunks, ",", 0, NULL);
661 
662  log_debug(LD_GENERAL, "Final bridge-ip-transports string: '%s'", the_string);
663 
664  done:
665  strmap_free(transport_counts, NULL);
666  SMARTLIST_FOREACH(transports_used, char *, s, tor_free(s));
667  smartlist_free(transports_used);
668  SMARTLIST_FOREACH(string_chunks, char *, s, tor_free(s));
669  smartlist_free(string_chunks);
670 
671  return the_string;
672 }
673 
674 /** Return a newly allocated comma-separated string containing statistics
675  * on network status downloads. The string contains the number of completed
676  * requests, timeouts, and still running requests as well as the download
677  * times by deciles and quartiles. Return NULL if we have not observed
678  * requests for long enough. */
679 static char *
680 geoip_get_dirreq_history(dirreq_type_t type)
681 {
682  char *result = NULL;
683  buf_t *buf = NULL;
684  smartlist_t *dirreq_completed = NULL;
685  uint32_t complete = 0, timeouts = 0, running = 0;
686  dirreq_map_entry_t **ptr, **next;
687  struct timeval now;
688 
689  tor_gettimeofday(&now);
690  dirreq_completed = smartlist_new();
691  for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) {
692  dirreq_map_entry_t *ent = *ptr;
693  if (ent->type != type) {
694  next = HT_NEXT(dirreqmap, &dirreq_map, ptr);
695  continue;
696  } else {
697  if (ent->completed) {
698  smartlist_add(dirreq_completed, ent);
699  complete++;
700  next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
701  } else {
702  if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT)
703  timeouts++;
704  else
705  running++;
706  next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
707  tor_free(ent);
708  }
709  }
710  }
711 #define DIR_REQ_GRANULARITY 4
712  complete = round_uint32_to_next_multiple_of(complete,
713  DIR_REQ_GRANULARITY);
715  DIR_REQ_GRANULARITY);
716  running = round_uint32_to_next_multiple_of(running,
717  DIR_REQ_GRANULARITY);
718  buf = buf_new_with_capacity(1024);
719  buf_add_printf(buf, "complete=%u,timeout=%u,"
720  "running=%u", complete, timeouts, running);
721 
722 #define MIN_DIR_REQ_RESPONSES 16
723  if (complete >= MIN_DIR_REQ_RESPONSES) {
724  uint32_t *dltimes;
725  /* We may have rounded 'completed' up. Here we want to use the
726  * real value. */
727  complete = smartlist_len(dirreq_completed);
728  dltimes = tor_calloc(complete, sizeof(uint32_t));
729  SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) {
730  uint32_t bytes_per_second;
731  uint32_t time_diff_ = (uint32_t) tv_mdiff(&ent->request_time,
732  &ent->completion_time);
733  if (time_diff_ == 0)
734  time_diff_ = 1; /* Avoid DIV/0; "instant" answers are impossible
735  * by law of nature or something, but a millisecond
736  * is a bit greater than "instantly" */
737  bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff_);
738  dltimes[ent_sl_idx] = bytes_per_second;
739  } SMARTLIST_FOREACH_END(ent);
740  median_uint32(dltimes, complete); /* sorts as a side effect. */
741  buf_add_printf(buf,
742  ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u,"
743  "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u",
744  dltimes[0],
745  dltimes[1*complete/10-1],
746  dltimes[2*complete/10-1],
747  dltimes[1*complete/4-1],
748  dltimes[3*complete/10-1],
749  dltimes[4*complete/10-1],
750  dltimes[5*complete/10-1],
751  dltimes[6*complete/10-1],
752  dltimes[7*complete/10-1],
753  dltimes[3*complete/4-1],
754  dltimes[8*complete/10-1],
755  dltimes[9*complete/10-1],
756  dltimes[complete-1]);
757  tor_free(dltimes);
758  }
759 
760  result = buf_extract(buf, NULL);
761 
762  SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent,
763  tor_free(ent));
764  smartlist_free(dirreq_completed);
765  buf_free(buf);
766  return result;
767 }
768 
769 /** Store a newly allocated comma-separated string in
770  * *<a>country_str</a> containing entries for all the countries from
771  * which we've seen enough clients connect as a bridge, directory
772  * server, or entry guard. The entry format is cc=num where num is the
773  * number of IPs we've seen connecting from that country, and cc is a
774  * lowercased country code. *<a>country_str</a> is set to NULL if
775  * we're not ready to export per country data yet.
776  *
777  * Store a newly allocated comma-separated string in <a>ipver_str</a>
778  * containing entries for clients connecting over IPv4 and IPv6. The
779  * format is family=num where num is the number of IPs we've seen
780  * connecting over that protocol family, and family is 'v4' or 'v6'.
781  *
782  * Return 0 on success and -1 if we're missing geoip data. */
783 int
785  char **country_str, char **ipver_str)
786 {
787  unsigned granularity = IP_GRANULARITY;
788  smartlist_t *entries = NULL;
789  int n_countries = geoip_get_n_countries();
790  int i;
791  clientmap_entry_t **cm_ent;
792  unsigned *counts = NULL;
793  unsigned total = 0;
794  unsigned ipv4_count = 0, ipv6_count = 0;
795 
796  if (!geoip_is_loaded(AF_INET) && !geoip_is_loaded(AF_INET6))
797  return -1;
798 
799  counts = tor_calloc(n_countries, sizeof(unsigned));
800  HT_FOREACH(cm_ent, clientmap, &client_history) {
801  int country;
802  if ((*cm_ent)->action != (int)action)
803  continue;
804  country = geoip_get_country_by_addr(&(*cm_ent)->addr);
805  if (country < 0)
806  country = 0; /** unresolved requests are stored at index 0. */
807  tor_assert(0 <= country && country < n_countries);
808  ++counts[country];
809  ++total;
810  switch (tor_addr_family(&(*cm_ent)->addr)) {
811  case AF_INET:
812  ipv4_count++;
813  break;
814  case AF_INET6:
815  ipv6_count++;
816  break;
817  }
818  }
819  if (ipver_str) {
820  smartlist_t *chunks = smartlist_new();
821  smartlist_add_asprintf(chunks, "v4=%u",
822  round_to_next_multiple_of(ipv4_count, granularity));
823  smartlist_add_asprintf(chunks, "v6=%u",
824  round_to_next_multiple_of(ipv6_count, granularity));
825  *ipver_str = smartlist_join_strings(chunks, ",", 0, NULL);
826  SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
827  smartlist_free(chunks);
828  }
829 
830  /* Don't record per country data if we haven't seen enough IPs. */
831  if (total < MIN_IPS_TO_NOTE_ANYTHING) {
832  tor_free(counts);
833  if (country_str)
834  *country_str = NULL;
835  return 0;
836  }
837 
838  /* Make a list of c_hist_t */
839  entries = smartlist_new();
840  for (i = 0; i < n_countries; ++i) {
841  unsigned c = counts[i];
842  const char *countrycode;
843  c_hist_t *ent;
844  /* Only report a country if it has a minimum number of IPs. */
845  if (c >= MIN_IPS_TO_NOTE_COUNTRY) {
846  c = round_to_next_multiple_of(c, granularity);
847  countrycode = geoip_get_country_name(i);
848  ent = tor_malloc(sizeof(c_hist_t));
849  strlcpy(ent->country, countrycode, sizeof(ent->country));
850  ent->total = c;
851  smartlist_add(entries, ent);
852  }
853  }
854  /* Sort entries. Note that we must do this _AFTER_ rounding, or else
855  * the sort order could leak info. */
856  smartlist_sort(entries, c_hist_compare_);
857 
858  if (country_str) {
859  smartlist_t *chunks = smartlist_new();
860  SMARTLIST_FOREACH(entries, c_hist_t *, ch, {
861  smartlist_add_asprintf(chunks, "%s=%u", ch->country, ch->total);
862  });
863  *country_str = smartlist_join_strings(chunks, ",", 0, NULL);
864  SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
865  smartlist_free(chunks);
866  }
867 
868  SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c));
869  smartlist_free(entries);
870  tor_free(counts);
871 
872  return 0;
873 }
874 
875 /** Return a newly allocated string holding the per-country request history
876  * for v3 network statuses in a format suitable for an extra-info document,
877  * or NULL on failure. */
878 char *
880 {
881  smartlist_t *entries, *strings;
882  char *result;
883  unsigned granularity = IP_GRANULARITY;
884 
885  entries = smartlist_new();
887  uint32_t tot = 0;
888  c_hist_t *ent;
889  if ((size_t)c_sl_idx < n_v3_ns_requests_len)
890  tot = n_v3_ns_requests[c_sl_idx];
891  else
892  tot = 0;
893  if (!tot)
894  continue;
895  ent = tor_malloc_zero(sizeof(c_hist_t));
896  strlcpy(ent->country, c->countrycode, sizeof(ent->country));
897  ent->total = round_to_next_multiple_of(tot, granularity);
898  smartlist_add(entries, ent);
899  } SMARTLIST_FOREACH_END(c);
900  smartlist_sort(entries, c_hist_compare_);
901 
902  strings = smartlist_new();
903  SMARTLIST_FOREACH(entries, c_hist_t *, ent, {
904  smartlist_add_asprintf(strings, "%s=%u", ent->country, ent->total);
905  });
906  result = smartlist_join_strings(strings, ",", 0, NULL);
907  SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp));
908  SMARTLIST_FOREACH(entries, c_hist_t *, ent, tor_free(ent));
909  smartlist_free(strings);
910  smartlist_free(entries);
911  return result;
912 }
913 
914 /** Start time of directory request stats or 0 if we're not collecting
915  * directory request statistics. */
916 static time_t start_of_dirreq_stats_interval;
917 
918 /** Initialize directory request stats. */
919 void
921 {
922  start_of_dirreq_stats_interval = now;
923 }
924 
925 /** Reset counters for dirreq stats. */
926 void
928 {
929  memset(n_v3_ns_requests, 0,
930  n_v3_ns_requests_len * sizeof(uint32_t));
931  {
932  clientmap_entry_t **ent, **next, *this;
933  for (ent = HT_START(clientmap, &client_history); ent != NULL;
934  ent = next) {
935  if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS) {
936  this = *ent;
937  next = HT_NEXT_RMV(clientmap, &client_history, ent);
938  clientmap_entry_free(this);
939  } else {
940  next = HT_NEXT(clientmap, &client_history, ent);
941  }
942  }
943  }
944  memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
945  {
946  dirreq_map_entry_t **ent, **next, *this;
947  for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
948  this = *ent;
949  next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
950  tor_free(this);
951  }
952  }
953  start_of_dirreq_stats_interval = now;
954 }
955 
956 /** Stop collecting directory request stats in a way that we can re-start
957  * doing so in geoip_dirreq_stats_init(). */
958 void
960 {
962 }
963 
964 /** Return a newly allocated string containing the dirreq statistics
965  * until <b>now</b>, or NULL if we're not collecting dirreq stats. Caller
966  * must ensure start_of_dirreq_stats_interval is in the past. */
967 char *
969 {
970  char t[ISO_TIME_LEN+1];
971  int i;
972  char *v3_ips_string = NULL, *v3_reqs_string = NULL,
973  *v3_direct_dl_string = NULL, *v3_tunneled_dl_string = NULL;
974  char *result = NULL;
975 
976  if (!start_of_dirreq_stats_interval)
977  return NULL; /* Not initialized. */
978 
979  tor_assert(now >= start_of_dirreq_stats_interval);
980 
981  format_iso_time(t, now);
983  v3_reqs_string = geoip_get_request_history();
984 
985 #define RESPONSE_GRANULARITY 8
986  for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) {
987  ns_v3_responses[i] = round_uint32_to_next_multiple_of(
988  ns_v3_responses[i], RESPONSE_GRANULARITY);
989  }
990 #undef RESPONSE_GRANULARITY
991 
992  v3_direct_dl_string = geoip_get_dirreq_history(DIRREQ_DIRECT);
993  v3_tunneled_dl_string = geoip_get_dirreq_history(DIRREQ_TUNNELED);
994 
995  /* Put everything together into a single string. */
996  tor_asprintf(&result, "dirreq-stats-end %s (%d s)\n"
997  "dirreq-v3-ips %s\n"
998  "dirreq-v3-reqs %s\n"
999  "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u,"
1000  "not-found=%u,not-modified=%u,busy=%u\n"
1001  "dirreq-v3-direct-dl %s\n"
1002  "dirreq-v3-tunneled-dl %s\n",
1003  t,
1004  (unsigned) (now - start_of_dirreq_stats_interval),
1005  v3_ips_string ? v3_ips_string : "",
1006  v3_reqs_string ? v3_reqs_string : "",
1007  ns_v3_responses[GEOIP_SUCCESS],
1008  ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS],
1009  ns_v3_responses[GEOIP_REJECT_UNAVAILABLE],
1010  ns_v3_responses[GEOIP_REJECT_NOT_FOUND],
1011  ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED],
1012  ns_v3_responses[GEOIP_REJECT_BUSY],
1013  v3_direct_dl_string ? v3_direct_dl_string : "",
1014  v3_tunneled_dl_string ? v3_tunneled_dl_string : "");
1015 
1016  /* Free partial strings. */
1017  tor_free(v3_ips_string);
1018  tor_free(v3_reqs_string);
1019  tor_free(v3_direct_dl_string);
1020  tor_free(v3_tunneled_dl_string);
1021 
1022  return result;
1023 }
1024 
1025 /** If 24 hours have passed since the beginning of the current dirreq
1026  * stats period, write dirreq stats to $DATADIR/stats/dirreq-stats
1027  * (possibly overwriting an existing file) and reset counters. Return
1028  * when we would next want to write dirreq stats or 0 if we never want to
1029  * write. */
1030 time_t
1032 {
1033  char *str = NULL;
1034 
1035  if (!start_of_dirreq_stats_interval)
1036  return 0; /* Not initialized. */
1037  if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now)
1038  goto done; /* Not ready to write. */
1039 
1040  /* Discard all items in the client history that are too old. */
1041  geoip_remove_old_clients(start_of_dirreq_stats_interval);
1042 
1043  /* Generate history string .*/
1044  str = geoip_format_dirreq_stats(now);
1045  if (! str)
1046  goto done;
1047 
1048  /* Write dirreq-stats string to disk. */
1049  if (!check_or_create_data_subdir("stats")) {
1050  write_to_data_subdir("stats", "dirreq-stats", str, "dirreq statistics");
1051  /* Reset measurement interval start. */
1053  }
1054 
1055  done:
1056  tor_free(str);
1057  return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL;
1058 }
1059 
1060 /** Start time of bridge stats or 0 if we're not collecting bridge
1061  * statistics. */
1062 static time_t start_of_bridge_stats_interval;
1063 
1064 /** Initialize bridge stats. */
1065 void
1067 {
1068  start_of_bridge_stats_interval = now;
1069 }
1070 
1071 /** Stop collecting bridge stats in a way that we can re-start doing so in
1072  * geoip_bridge_stats_init(). */
1073 void
1075 {
1076  client_history_clear();
1077  start_of_bridge_stats_interval = 0;
1078 }
1079 
1080 /** Validate a bridge statistics string as it would be written to a
1081  * current extra-info descriptor. Return 1 if the string is valid and
1082  * recent enough, or 0 otherwise. */
1083 static int
1084 validate_bridge_stats(const char *stats_str, time_t now)
1085 {
1086  char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1],
1087  *eos;
1088 
1089  const char *BRIDGE_STATS_END = "bridge-stats-end ";
1090  const char *BRIDGE_IPS = "bridge-ips ";
1091  const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n";
1092  const char *BRIDGE_TRANSPORTS = "bridge-ip-transports ";
1093  const char *BRIDGE_TRANSPORTS_EMPTY_LINE = "bridge-ip-transports\n";
1094  const char *tmp;
1095  time_t stats_end_time;
1096  int seconds;
1097  tor_assert(stats_str);
1098 
1099  /* Parse timestamp and number of seconds from
1100  "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */
1101  tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END);
1102  if (!tmp)
1103  return 0;
1104  tmp += strlen(BRIDGE_STATS_END);
1105 
1106  if (strlen(tmp) < ISO_TIME_LEN + 6)
1107  return 0;
1108  strlcpy(stats_end_str, tmp, sizeof(stats_end_str));
1109  if (parse_iso_time(stats_end_str, &stats_end_time) < 0)
1110  return 0;
1111  if (stats_end_time < now - (25*60*60) ||
1112  stats_end_time > now + (1*60*60))
1113  return 0;
1114  seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10);
1115  if (!eos || seconds < 23*60*60)
1116  return 0;
1117  format_iso_time(stats_start_str, stats_end_time - seconds);
1118 
1119  /* Parse: "bridge-ips CC=N,CC=N,..." */
1120  tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS);
1121  if (!tmp) {
1122  /* Look if there is an empty "bridge-ips" line */
1123  tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE);
1124  if (!tmp)
1125  return 0;
1126  }
1127 
1128  /* Parse: "bridge-ip-transports PT=N,PT=N,..." */
1129  tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS);
1130  if (!tmp) {
1131  /* Look if there is an empty "bridge-ip-transports" line */
1132  tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS_EMPTY_LINE);
1133  if (!tmp)
1134  return 0;
1135  }
1136 
1137  return 1;
1138 }
1139 
1140 /** Most recent bridge statistics formatted to be written to extra-info
1141  * descriptors. */
1142 static char *bridge_stats_extrainfo = NULL;
1143 
1144 /** Return a newly allocated string holding our bridge usage stats by country
1145  * in a format suitable for inclusion in an extrainfo document. Return NULL on
1146  * failure. */
1147 char *
1149 {
1150  char *out = NULL;
1151  char *country_data = NULL, *ipver_data = NULL, *transport_data = NULL;
1152  long duration = now - start_of_bridge_stats_interval;
1153  char written[ISO_TIME_LEN+1];
1154 
1155  if (duration < 0)
1156  return NULL;
1157  if (!start_of_bridge_stats_interval)
1158  return NULL; /* Not initialized. */
1159 
1160  format_iso_time(written, now);
1161  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
1162  transport_data = geoip_get_transport_history();
1163 
1164  tor_asprintf(&out,
1165  "bridge-stats-end %s (%ld s)\n"
1166  "bridge-ips %s\n"
1167  "bridge-ip-versions %s\n"
1168  "bridge-ip-transports %s\n",
1169  written, duration,
1170  country_data ? country_data : "",
1171  ipver_data ? ipver_data : "",
1172  transport_data ? transport_data : "");
1173  tor_free(country_data);
1174  tor_free(ipver_data);
1175  tor_free(transport_data);
1176 
1177  return out;
1178 }
1179 
1180 /** Return a newly allocated string holding our bridge usage stats by country
1181  * in a format suitable for the answer to a controller request. Return NULL on
1182  * failure. */
1183 static char *
1184 format_bridge_stats_controller(time_t now)
1185 {
1186  char *out = NULL, *country_data = NULL, *ipver_data = NULL;
1187  char started[ISO_TIME_LEN+1];
1188  (void) now;
1189 
1190  format_iso_time(started, start_of_bridge_stats_interval);
1191  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
1192 
1193  tor_asprintf(&out,
1194  "TimeStarted=\"%s\" CountrySummary=%s IPVersions=%s",
1195  started,
1196  country_data ? country_data : "",
1197  ipver_data ? ipver_data : "");
1198  tor_free(country_data);
1199  tor_free(ipver_data);
1200  return out;
1201 }
1202 
1203 /** Return a newly allocated string holding our bridge usage stats by
1204  * country in a format suitable for inclusion in our heartbeat
1205  * message. Return NULL on failure. */
1206 char *
1208 {
1209  const int n_hours = 6;
1210  char *out = NULL;
1211  int n_clients = 0;
1212  clientmap_entry_t **ent;
1213  unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 );
1214 
1215  if (!start_of_bridge_stats_interval)
1216  return NULL; /* Not initialized. */
1217 
1218  /* count unique IPs */
1219  HT_FOREACH(ent, clientmap, &client_history) {
1220  /* only count directly connecting clients */
1221  if ((*ent)->action != GEOIP_CLIENT_CONNECT)
1222  continue;
1223  if ((*ent)->last_seen_in_minutes < cutoff)
1224  continue;
1225  n_clients++;
1226  }
1227 
1228  tor_asprintf(&out, "Heartbeat: "
1229  "In the last %d hours, I have seen %d unique clients.",
1230  n_hours,
1231  n_clients);
1232 
1233  return out;
1234 }
1235 
1236 /** Write bridge statistics to $DATADIR/stats/bridge-stats and return
1237  * when we should next try to write statistics. */
1238 time_t
1240 {
1241  char *val = NULL;
1242 
1243  /* Check if 24 hours have passed since starting measurements. */
1244  if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL)
1245  return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
1246 
1247  /* Discard all items in the client history that are too old. */
1248  geoip_remove_old_clients(start_of_bridge_stats_interval);
1249 
1250  /* Generate formatted string */
1251  val = geoip_format_bridge_stats(now);
1252  if (val == NULL)
1253  goto done;
1254 
1255  /* Update the stored value. */
1256  tor_free(bridge_stats_extrainfo);
1257  bridge_stats_extrainfo = val;
1258  start_of_bridge_stats_interval = now;
1259 
1260  /* Write it to disk. */
1261  if (!check_or_create_data_subdir("stats")) {
1262  write_to_data_subdir("stats", "bridge-stats",
1263  bridge_stats_extrainfo, "bridge statistics");
1264 
1265  /* Tell the controller, "hey, there are clients!" */
1266  {
1267  char *controller_str = format_bridge_stats_controller(now);
1268  if (controller_str)
1269  control_event_clients_seen(controller_str);
1270  tor_free(controller_str);
1271  }
1272  }
1273 
1274  done:
1275  return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
1276 }
1277 
1278 /** Try to load the most recent bridge statistics from disk, unless we
1279  * have finished a measurement interval lately, and check whether they
1280  * are still recent enough. */
1281 static void
1282 load_bridge_stats(time_t now)
1283 {
1284  char *fname, *contents;
1285  if (bridge_stats_extrainfo)
1286  return;
1287 
1288  fname = get_datadir_fname2("stats", "bridge-stats");
1289  contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
1290  if (contents && validate_bridge_stats(contents, now)) {
1291  bridge_stats_extrainfo = contents;
1292  } else {
1293  tor_free(contents);
1294  }
1295 
1296  tor_free(fname);
1297 }
1298 
1299 /** Return most recent bridge statistics for inclusion in extra-info
1300  * descriptors, or NULL if we don't have recent bridge statistics. */
1301 const char *
1303 {
1304  load_bridge_stats(now);
1305  return bridge_stats_extrainfo;
1306 }
1307 
1308 /** Return a new string containing the recent bridge statistics to be returned
1309  * to controller clients, or NULL if we don't have any bridge statistics. */
1310 char *
1312 {
1313  return format_bridge_stats_controller(now);
1314 }
1315 
1316 /** Start time of entry stats or 0 if we're not collecting entry
1317  * statistics. */
1318 static time_t start_of_entry_stats_interval;
1319 
1320 /** Initialize entry stats. */
1321 void
1323 {
1324  start_of_entry_stats_interval = now;
1325 }
1326 
1327 /** Reset counters for entry stats. */
1328 void
1330 {
1331  client_history_clear();
1332  start_of_entry_stats_interval = now;
1333 }
1334 
1335 /** Stop collecting entry stats in a way that we can re-start doing so in
1336  * geoip_entry_stats_init(). */
1337 void
1339 {
1341 }
1342 
1343 /** Return a newly allocated string containing the entry statistics
1344  * until <b>now</b>, or NULL if we're not collecting entry stats. Caller
1345  * must ensure start_of_entry_stats_interval lies in the past. */
1346 char *
1348 {
1349  char t[ISO_TIME_LEN+1];
1350  char *data = NULL;
1351  char *result;
1352 
1353  if (!start_of_entry_stats_interval)
1354  return NULL; /* Not initialized. */
1355 
1356  tor_assert(now >= start_of_entry_stats_interval);
1357 
1359  format_iso_time(t, now);
1360  tor_asprintf(&result,
1361  "entry-stats-end %s (%u s)\n"
1362  "entry-ips %s\n",
1363  t, (unsigned) (now - start_of_entry_stats_interval),
1364  data ? data : "");
1365  tor_free(data);
1366  return result;
1367 }
1368 
1369 /** If 24 hours have passed since the beginning of the current entry stats
1370  * period, write entry stats to $DATADIR/stats/entry-stats (possibly
1371  * overwriting an existing file) and reset counters. Return when we would
1372  * next want to write entry stats or 0 if we never want to write. */
1373 time_t
1375 {
1376  char *str = NULL;
1377 
1378  if (!start_of_entry_stats_interval)
1379  return 0; /* Not initialized. */
1380  if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now)
1381  goto done; /* Not ready to write. */
1382 
1383  /* Discard all items in the client history that are too old. */
1384  geoip_remove_old_clients(start_of_entry_stats_interval);
1385 
1386  /* Generate history string .*/
1387  str = geoip_format_entry_stats(now);
1388 
1389  /* Write entry-stats string to disk. */
1390  if (!check_or_create_data_subdir("stats")) {
1391  write_to_data_subdir("stats", "entry-stats", str, "entry statistics");
1392 
1393  /* Reset measurement interval start. */
1395  }
1396 
1397  done:
1398  tor_free(str);
1399  return start_of_entry_stats_interval + WRITE_STATS_INTERVAL;
1400 }
1401 
1402 /** Release all storage held in this file. */
1403 void
1405 {
1406  {
1407  clientmap_entry_t **ent, **next, *this;
1408  for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) {
1409  this = *ent;
1410  next = HT_NEXT_RMV(clientmap, &client_history, ent);
1411  clientmap_entry_free(this);
1412  }
1413  HT_CLEAR(clientmap, &client_history);
1414  }
1415  {
1416  dirreq_map_entry_t **ent, **next, *this;
1417  for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
1418  this = *ent;
1419  next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
1420  tor_free(this);
1421  }
1422  HT_CLEAR(dirreqmap, &dirreq_map);
1423  }
1424 
1425  tor_free(bridge_stats_extrainfo);
1426  tor_free(n_v3_ns_requests);
1427 }
country_t
int16_t country_t
Definition: country.h:17
geoip_country_t
Definition: geoip.h:33
tor_addr_compare
int tor_addr_compare(const tor_addr_t *addr1, const tor_addr_t *addr2, tor_addr_comparison_t how)
Definition: address.c:984
timeouts
Definition: timeout.c:215
tor_free
#define tor_free(p)
Definition: malloc.h:52
buf_extract
char * buf_extract(buf_t *buf, size_t *sz_out)
Definition: buffers.c:592
counts
static conn_counts_t counts
Definition: connstats.c:72
buf_new_with_capacity
buf_t * buf_new_with_capacity(size_t size)
Definition: buffers.c:356
find_str_at_start_of_line
const char * find_str_at_start_of_line(const char *haystack, const char *needle)
Definition: util_string.c:400
tor_free_
void tor_free_(void *mem)
Definition: malloc.c:227
tor_addr_family
static sa_family_t tor_addr_family(const tor_addr_t *a)
Definition: address.h:187
tor_addr_t
Definition: address.h:69
geoip_get_transport_history
char * geoip_get_transport_history(void)
Definition: geoip_stats.c:579
dnsserv.h
Header file for dnsserv.c.
smartlist_add_strdup
void smartlist_add_strdup(struct smartlist_t *sl, const char *string)
Definition: smartlist_core.c:137
or_options_t::BridgeRelay
int BridgeRelay
Definition: or_options_st.h:271
geoip_bridge_stats_term
void geoip_bridge_stats_term(void)
Definition: geoip_stats.c:1074
or_options_t::DirReqStatistics
int DirReqStatistics
Definition: or_options_st.h:649
geoip_dirreq_stats_term
void geoip_dirreq_stats_term(void)
Definition: geoip_stats.c:959
tor_assert
#define tor_assert(expr)
Definition: util_bug.h:102
LD_BUG
#define LD_BUG
Definition: log.h:86
geoip_format_entry_stats
char * geoip_format_entry_stats(time_t now)
Definition: geoip_stats.c:1347
geoip_client_action_t
geoip_client_action_t
Definition: geoip_stats.h:22
geoip_note_ns_response
void geoip_note_ns_response(geoip_ns_response_t response)
Definition: geoip_stats.c:395
format_iso_time
void format_iso_time(char *buf, time_t t)
Definition: time_fmt.c:295
LD_GENERAL
#define LD_GENERAL
Definition: log.h:62
geoip_format_dirreq_stats
char * geoip_format_dirreq_stats(time_t now)
Definition: geoip_stats.c:968
geoip_is_loaded
int geoip_is_loaded(sa_family_t family)
Definition: geoip.c:458
GEOIP_REJECT_NOT_MODIFIED
@ GEOIP_REJECT_NOT_MODIFIED
Definition: geoip_stats.h:41
dirreq_map_entry_t
Definition: geoip_stats.c:450
tor_reallocarray_
void * tor_reallocarray_(void *ptr, size_t sz1, size_t sz2)
Definition: malloc.c:146
dirreq_state_t
dirreq_state_t
Definition: geoip_stats.h:56
smartlist_add
void smartlist_add(smartlist_t *sl, void *element)
Definition: smartlist_core.c:117
check_or_create_data_subdir
int check_or_create_data_subdir(const char *subdir)
Definition: config.c:7046
GEOIP_SUCCESS
@ GEOIP_SUCCESS
Definition: geoip_stats.h:32
COUNTRY_MAX
#define COUNTRY_MAX
Definition: country.h:20
control_event_clients_seen
void control_event_clients_seen(const char *controller_str)
Definition: control_events.c:1881
smartlist_new
smartlist_t * smartlist_new(void)
Definition: smartlist_core.c:26
round_uint64_to_next_multiple_of
uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor)
Definition: muldiv.c:50
parse_iso_time
int parse_iso_time(const char *cp, time_t *t)
Definition: time_fmt.c:392
GEOIP_REJECT_NOT_FOUND
@ GEOIP_REJECT_NOT_FOUND
Definition: geoip_stats.h:39
SMARTLIST_FOREACH
#define SMARTLIST_FOREACH(sl, type, var, cmd)
Definition: smartlist_foreach.h:112
GEOIP_REJECT_BUSY
@ GEOIP_REJECT_BUSY
Definition: geoip_stats.h:43
geoip_dirreq_stats_init
void geoip_dirreq_stats_init(time_t now)
Definition: geoip_stats.c:920
tvdiff.h
Header for tvdiff.c.
geoip_stats.h
Header file for geoip_stats.c.
clientmap_entry_t
Definition: geoip_stats.h:79
GEOIP_REJECT_NOT_ENOUGH_SIGS
@ GEOIP_REJECT_NOT_ENOUGH_SIGS
Definition: geoip_stats.h:35
geoip_bridge_stats_write
time_t geoip_bridge_stats_write(time_t now)
Definition: geoip_stats.c:1239
should_record_bridge_info
int should_record_bridge_info(const or_options_t *options)
Definition: geoip_stats.c:112
geoip_get_bridge_stats_controller
char * geoip_get_bridge_stats_controller(time_t)
Definition: geoip_stats.c:1311
or_options_t::BridgeAuthoritativeDir
int BridgeAuthoritativeDir
Definition: or_options_st.h:242
strcmp_opt
int strcmp_opt(const char *s1, const char *s2)
Definition: util_string.c:197
geoip_start_dirreq
void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, dirreq_type_t type)
Definition: geoip_stats.c:531
c_hist_t::country
char country[3]
Definition: geoip_stats.c:421
geoip_reset_entry_stats
void geoip_reset_entry_stats(time_t now)
Definition: geoip_stats.c:1329
geoip.h
Header file for geoip.c.
geoip_format_bridge_stats
char * geoip_format_bridge_stats(time_t now)
Definition: geoip_stats.c:1148
or_options_t::BridgeRecordUsageByCountry
int BridgeRecordUsageByCountry
Definition: or_options_st.h:825
RFTS_IGNORE_MISSING
#define RFTS_IGNORE_MISSING
Definition: files.h:101
GEOIP_CLIENT_CONNECT
@ GEOIP_CLIENT_CONNECT
Definition: geoip_stats.h:24
round_to_next_multiple_of
unsigned round_to_next_multiple_of(unsigned number, unsigned divisor)
Definition: muldiv.c:21
or_options_t::EntryStatistics
int EntryStatistics
Definition: or_options_st.h:664
tor_addr_hash
uint64_t tor_addr_hash(const tor_addr_t *addr)
Definition: address.c:1123
geoip_remove_old_clients
void geoip_remove_old_clients(time_t cutoff)
Definition: geoip_stats.c:293
routerlist.h
Header file for routerlist.c.
GEOIP_REJECT_UNAVAILABLE
@ GEOIP_REJECT_UNAVAILABLE
Definition: geoip_stats.h:37
DIRREQ_IS_FOR_NETWORK_STATUS
@ DIRREQ_IS_FOR_NETWORK_STATUS
Definition: geoip_stats.h:60
tor_gettimeofday
void tor_gettimeofday(struct timeval *timeval)
Definition: tor_gettimeofday.c:42
buffers.h
Header file for buffers.c.
geoip_get_country_by_addr
int geoip_get_country_by_addr(const tor_addr_t *addr)
Definition: geoip.c:424
round_uint32_to_next_multiple_of
uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor)
Definition: muldiv.c:35
HT_HEAD
typedef HT_HEAD(hs_service_ht, hs_service_t) hs_service_ht
DIRREQ_FLUSHING_DIR_CONN_FINISHED
@ DIRREQ_FLUSHING_DIR_CONN_FINISHED
Definition: geoip_stats.h:64
geoip_entry_stats_write
time_t geoip_entry_stats_write(time_t now)
Definition: geoip_stats.c:1374
fmt_addr
#define fmt_addr(a)
Definition: address.h:239
control_events.h
Header file for control_events.c.
get_options
const or_options_t * get_options(void)
Definition: config.c:919
geoip_note_client_seen
void geoip_note_client_seen(geoip_client_action_t action, const tor_addr_t *addr, const char *transport_name, time_t now)
Definition: geoip_stats.c:229
geoip_bridge_stats_init
void geoip_bridge_stats_init(time_t now)
Definition: geoip_stats.c:1066
write_to_data_subdir
int write_to_data_subdir(const char *subdir, const char *fname, const char *str, const char *descr)
Definition: config.c:7065
smartlist_sort_strings
void smartlist_sort_strings(smartlist_t *sl)
Definition: smartlist.c:549
geoip_reset_dirreq_stats
void geoip_reset_dirreq_stats(time_t now)
Definition: geoip_stats.c:927
geoip_stats_free_all
void geoip_stats_free_all(void)
Definition: geoip_stats.c:1404
IF_BUG_ONCE
#define IF_BUG_ONCE(cond)
Definition: util_bug.h:246
geoip_change_dirreq_state
void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, dirreq_state_t new_state)
Definition: geoip_stats.c:552
geoip_get_bridge_stats_extrainfo
const char * geoip_get_bridge_stats_extrainfo(time_t)
Definition: geoip_stats.c:1302
tv_mdiff
long tv_mdiff(const struct timeval *start, const struct timeval *end)
Definition: tvdiff.c:102
timeval
Definition: compat_time.h:151
tor_asprintf
int tor_asprintf(char **strp, const char *fmt,...)
Definition: printf.c:75
GEOIP_CLIENT_NETWORKSTATUS
@ GEOIP_CLIENT_NETWORKSTATUS
Definition: geoip_stats.h:26
SMARTLIST_FOREACH_BEGIN
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
Definition: smartlist_foreach.h:78
format_client_stats_heartbeat
char * format_client_stats_heartbeat(time_t now)
Definition: geoip_stats.c:1207
geoip_entry_stats_init
void geoip_entry_stats_init(time_t now)
Definition: geoip_stats.c:1322
geoip_ns_response_t
geoip_ns_response_t
Definition: geoip_stats.h:30
geoip_get_client_history
int geoip_get_client_history(geoip_client_action_t action, char **country_str, char **ipver_str)
Definition: geoip_stats.c:784
HT_PROTOTYPE
HT_PROTOTYPE(hs_circuitmap_ht, circuit_t, hs_circuitmap_node, hs_circuit_hash_token, hs_circuits_have_same_token)
order.h
Header for order.c.
config.h
Header file for config.c.
geoip_entry_stats_term
void geoip_entry_stats_term(void)
Definition: geoip_stats.c:1338
geoip_get_request_history
char * geoip_get_request_history(void)
Definition: geoip_stats.c:879
smartlist_add_asprintf
void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern,...)
Definition: smartlist.c:36
c_hist_t::total
unsigned total
Definition: geoip_stats.c:422
or_options_t
Definition: or_options_st.h:64
buf_add_printf
void buf_add_printf(buf_t *buf, const char *format,...)
Definition: buffers.c:568
geoip_dirreq_stats_write
time_t geoip_dirreq_stats_write(time_t now)
Definition: geoip_stats.c:1031
geoip_get_n_countries
int geoip_get_n_countries(void)
Definition: geoip.c:437
smartlist_sort
void smartlist_sort(smartlist_t *sl, int(*compare)(const void **a, const void **b))
Definition: smartlist.c:334
smartlist_t
Definition: smartlist_core.h:26
geoip_get_countries
const smartlist_t * geoip_get_countries(void)
Definition: geoip.c:89
smartlist_join_strings
char * smartlist_join_strings(smartlist_t *sl, const char *join, int terminate, size_t *len_out)
Definition: smartlist.c:279
geoip_get_country_name
const char * geoip_get_country_name(country_t num)
Definition: geoip.c:447
tor_addr_copy
void tor_addr_copy(tor_addr_t *dest, const tor_addr_t *src)
Definition: address.c:933
DIRREQ_CHANNEL_BUFFER_FLUSHED
@ DIRREQ_CHANNEL_BUFFER_FLUSHED
Definition: geoip_stats.h:73
dirreq_type_t
dirreq_type_t
Definition: geoip_stats.h:49
c_hist_t
Definition: geoip_stats.c:420
clientmap_entry_t::last_seen_in_minutes
unsigned int last_seen_in_minutes
Definition: geoip_stats.h:90
or.h
Master header file for Tor-specific functionality.