Line data Source code
1 : /* Copyright (c) 2007-2021, The Tor Project, Inc. */
2 : /* See LICENSE for licensing information */
3 :
4 : /**
5 : * \file geoip.c
6 : * \brief Functions related to maintaining an IP-to-country database;
7 : * to summarizing client connections by country to entry guards, bridges,
8 : * and directory servers; and for statistics on answering network status
9 : * requests.
10 : *
11 : * There are two main kinds of functions in this module: geoip functions,
12 : * which map groups of IPv4 and IPv6 addresses to country codes, and
13 : * statistical functions, which collect statistics about different kinds of
14 : * per-country usage.
15 : *
16 : * The geoip lookup tables are implemented as sorted lists of disjoint address
17 : * ranges, each mapping to a singleton geoip_country_t. These country objects
18 : * are also indexed by their names in a hashtable.
19 : *
20 : * The tables are populated from disk at startup by the geoip_load_file()
21 : * function. For more information on the file format they read, see that
22 : * function. See the scripts and the README file in src/config for more
23 : * information about how those files are generated.
24 : *
25 : * Tor uses GeoIP information in order to implement user requests (such as
26 : * ExcludeNodes {cc}), and to keep track of how much usage relays are getting
27 : * for each country.
28 : */
29 :
30 : #include "core/or/or.h"
31 :
32 : #include "ht.h"
33 : #include "lib/buf/buffers.h"
34 : #include "app/config/config.h"
35 : #include "feature/control/control_events.h"
36 : #include "feature/client/dnsserv.h"
37 : #include "core/or/dos.h"
38 : #include "lib/geoip/geoip.h"
39 : #include "feature/stats/geoip_stats.h"
40 : #include "feature/nodelist/routerlist.h"
41 :
42 : #include "lib/container/order.h"
43 : #include "lib/time/tvdiff.h"
44 :
45 : /** Number of entries in n_v3_ns_requests */
46 : static size_t n_v3_ns_requests_len = 0;
47 : /** Array, indexed by country index, of number of v3 networkstatus requests
48 : * received from that country */
49 : static uint32_t *n_v3_ns_requests;
50 :
51 : /* Total size in bytes of the geoip client history cache. Used by the OOM
52 : * handler. */
53 : static size_t geoip_client_history_cache_size;
54 :
55 : /* Increment the geoip client history cache size counter with the given bytes.
56 : * This prevents an overflow and set it to its maximum in that case. */
57 : static inline void
58 213 : geoip_increment_client_history_cache_size(size_t bytes)
59 : {
60 : /* This is shockingly high, lets log it so it can be reported. */
61 213 : IF_BUG_ONCE(geoip_client_history_cache_size > (SIZE_MAX - bytes)) {
62 0 : geoip_client_history_cache_size = SIZE_MAX;
63 0 : return;
64 : }
65 213 : geoip_client_history_cache_size += bytes;
66 : }
67 :
68 : /* Decrement the geoip client history cache size counter with the given bytes.
69 : * This prevents an underflow and set it to 0 in that case. */
70 : static inline void
71 206 : geoip_decrement_client_history_cache_size(size_t bytes)
72 : {
73 : /* Going below 0 means that we either allocated an entry without
74 : * incrementing the counter or we have different sizes when allocating and
75 : * freeing. It shouldn't happened so log it. */
76 206 : IF_BUG_ONCE(geoip_client_history_cache_size < bytes) {
77 0 : geoip_client_history_cache_size = 0;
78 0 : return;
79 : }
80 206 : geoip_client_history_cache_size -= bytes;
81 : }
82 :
83 : /** Add 1 to the count of v3 ns requests received from <b>country</b>. */
84 : static void
85 5 : increment_v3_ns_request(country_t country)
86 : {
87 5 : if (country < 0)
88 : return;
89 :
90 5 : if ((size_t)country >= n_v3_ns_requests_len) {
91 : /* We need to reallocate the array. */
92 2 : size_t new_len;
93 2 : if (n_v3_ns_requests_len == 0)
94 : new_len = 256;
95 : else
96 0 : new_len = n_v3_ns_requests_len * 2;
97 2 : if (new_len <= (size_t)country)
98 0 : new_len = ((size_t)country)+1;
99 2 : n_v3_ns_requests = tor_reallocarray(n_v3_ns_requests, new_len,
100 : sizeof(uint32_t));
101 2 : memset(n_v3_ns_requests + n_v3_ns_requests_len, 0,
102 2 : sizeof(uint32_t)*(new_len - n_v3_ns_requests_len));
103 2 : n_v3_ns_requests_len = new_len;
104 : }
105 :
106 5 : n_v3_ns_requests[country] += 1;
107 : }
108 :
109 : /** Return 1 if we should collect geoip stats on bridge users, and
110 : * include them in our extrainfo descriptor. Else return 0. */
111 : int
112 14 : should_record_bridge_info(const or_options_t *options)
113 : {
114 14 : return options->BridgeRelay && options->BridgeRecordUsageByCountry;
115 : }
116 :
117 : /** Largest allowable value for last_seen_in_minutes. (It's a 30-bit field,
118 : * so it can hold up to (1u<<30)-1, or 0x3fffffffu.
119 : */
120 : #define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu
121 :
122 : /** Map from client IP address to last time seen. */
123 : static HT_HEAD(clientmap, clientmap_entry_t) client_history =
124 : HT_INITIALIZER();
125 :
126 : /** Hashtable helper: compute a hash of a clientmap_entry_t. */
127 : static inline unsigned
128 1436 : clientmap_entry_hash(const clientmap_entry_t *a)
129 : {
130 1436 : unsigned h = (unsigned) tor_addr_hash(&a->addr);
131 :
132 1436 : if (a->transport_name)
133 330 : h += (unsigned) siphash24g(a->transport_name, strlen(a->transport_name));
134 :
135 1436 : return h;
136 : }
137 : /** Hashtable helper: compare two clientmap_entry_t values for equality. */
138 : static inline int
139 1075 : clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b)
140 : {
141 1075 : if (strcmp_opt(a->transport_name, b->transport_name))
142 : return 0;
143 :
144 1056 : return !tor_addr_compare(&a->addr, &b->addr, CMP_EXACT) &&
145 1010 : a->action == b->action;
146 : }
147 :
148 15536 : HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
149 : clientmap_entries_eq);
150 793 : HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
151 : clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_);
152 :
153 : #define clientmap_entry_free(ent) \
154 : FREE_AND_NULL(clientmap_entry_t, clientmap_entry_free_, ent)
155 :
156 : /** Return the size of a client map entry. */
157 : static inline size_t
158 420 : clientmap_entry_size(const clientmap_entry_t *ent)
159 : {
160 420 : tor_assert(ent);
161 1170 : return (sizeof(clientmap_entry_t) +
162 420 : (ent->transport_name ? strlen(ent->transport_name) : 0));
163 : }
164 :
165 : /** Free all storage held by <b>ent</b>. */
166 : static void
167 206 : clientmap_entry_free_(clientmap_entry_t *ent)
168 : {
169 206 : if (!ent)
170 : return;
171 :
172 : /* This entry is about to be freed so pass it to the DoS subsystem to see if
173 : * any actions can be taken about it. */
174 206 : dos_geoip_entry_about_to_free(ent);
175 206 : geoip_decrement_client_history_cache_size(clientmap_entry_size(ent));
176 :
177 206 : tor_free(ent->transport_name);
178 206 : tor_free(ent);
179 : }
180 :
181 : /* Return a newly allocated clientmap entry with the given action and address
182 : * that are mandatory. The transport_name can be optional. This can't fail. */
183 : static clientmap_entry_t *
184 213 : clientmap_entry_new(geoip_client_action_t action, const tor_addr_t *addr,
185 : const char *transport_name)
186 : {
187 213 : clientmap_entry_t *entry;
188 :
189 213 : tor_assert(action == GEOIP_CLIENT_CONNECT ||
190 : action == GEOIP_CLIENT_NETWORKSTATUS);
191 213 : tor_assert(addr);
192 :
193 213 : entry = tor_malloc_zero(sizeof(clientmap_entry_t));
194 213 : entry->action = action;
195 213 : tor_addr_copy(&entry->addr, addr);
196 213 : if (transport_name) {
197 165 : entry->transport_name = tor_strdup(transport_name);
198 : }
199 : /* Initialize the DoS object. */
200 213 : dos_geoip_entry_init(entry);
201 :
202 : /* Allocated and initialized, note down its size for the OOM handler. */
203 213 : geoip_increment_client_history_cache_size(clientmap_entry_size(entry));
204 :
205 213 : return entry;
206 : }
207 :
208 : /** Clear history of connecting clients used by entry and bridge stats. */
209 : static void
210 6 : client_history_clear(void)
211 : {
212 6 : clientmap_entry_t **ent, **next, *this;
213 199 : for (ent = HT_START(clientmap, &client_history); ent != NULL;
214 : ent = next) {
215 193 : if ((*ent)->action == GEOIP_CLIENT_CONNECT) {
216 193 : this = *ent;
217 193 : next = HT_NEXT_RMV(clientmap, &client_history, ent);
218 193 : clientmap_entry_free(this);
219 : } else {
220 0 : next = HT_NEXT(clientmap, &client_history, ent);
221 : }
222 : }
223 6 : }
224 :
225 : /** Note that we've seen a client connect from the IP <b>addr</b>
226 : * at time <b>now</b>. Ignored by all but bridges and directories if
227 : * configured accordingly. */
228 : void
229 242 : geoip_note_client_seen(geoip_client_action_t action,
230 : const tor_addr_t *addr,
231 : const char *transport_name,
232 : time_t now)
233 : {
234 242 : const or_options_t *options = get_options();
235 242 : clientmap_entry_t *ent;
236 :
237 242 : if (action == GEOIP_CLIENT_CONNECT) {
238 : /* Only remember statistics if the DoS mitigation subsystem is enabled. If
239 : * not, only if as entry guard or as bridge. */
240 237 : if (!dos_enabled()) {
241 231 : if (!options->EntryStatistics && !should_record_bridge_info(options)) {
242 : return;
243 : }
244 : }
245 : } else {
246 : /* Only gather directory-request statistics if configured, and
247 : * forcibly disable them on bridge authorities. */
248 5 : if (!options->DirReqStatistics || options->BridgeAuthoritativeDir)
249 : return;
250 : }
251 :
252 319 : log_debug(LD_GENERAL, "Seen client from '%s' with transport '%s'.",
253 : safe_str_client(fmt_addr((addr))),
254 : transport_name ? transport_name : "<no transport>");
255 :
256 242 : ent = geoip_lookup_client(addr, transport_name, action);
257 242 : if (! ent) {
258 213 : ent = clientmap_entry_new(action, addr, transport_name);
259 213 : HT_INSERT(clientmap, &client_history, ent);
260 : }
261 242 : if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0)
262 242 : ent->last_seen_in_minutes = (unsigned)(now/60);
263 : else
264 0 : ent->last_seen_in_minutes = 0;
265 :
266 242 : if (action == GEOIP_CLIENT_NETWORKSTATUS) {
267 5 : int country_idx = geoip_get_country_by_addr(addr);
268 5 : if (country_idx < 0)
269 : country_idx = 0; /** unresolved requests are stored at index 0. */
270 5 : IF_BUG_ONCE(country_idx > COUNTRY_MAX) {
271 : return;
272 : }
273 5 : increment_v3_ns_request((country_t) country_idx);
274 : }
275 : }
276 :
277 : /** HT_FOREACH helper: remove a clientmap_entry_t from the hashtable if it's
278 : * older than a certain time. */
279 : static int
280 29 : remove_old_client_helper_(struct clientmap_entry_t *ent, void *_cutoff)
281 : {
282 29 : time_t cutoff = *(time_t*)_cutoff / 60;
283 29 : if (ent->last_seen_in_minutes < cutoff) {
284 9 : clientmap_entry_free(ent);
285 9 : return 1;
286 : } else {
287 : return 0;
288 : }
289 : }
290 :
291 : /** Forget about all clients that haven't connected since <b>cutoff</b>. */
292 : void
293 1 : geoip_remove_old_clients(time_t cutoff)
294 : {
295 1 : clientmap_HT_FOREACH_FN(&client_history,
296 : remove_old_client_helper_,
297 : &cutoff);
298 1 : }
299 :
300 : /* Return a client entry object matching the given address, transport name and
301 : * geoip action from the clientmap. NULL if not found. The transport_name can
302 : * be NULL. */
303 : clientmap_entry_t *
304 1223 : geoip_lookup_client(const tor_addr_t *addr, const char *transport_name,
305 : geoip_client_action_t action)
306 : {
307 1223 : clientmap_entry_t lookup;
308 :
309 1223 : tor_assert(addr);
310 :
311 : /* We always look for a client connection with no transport. */
312 1223 : tor_addr_copy(&lookup.addr, addr);
313 1223 : lookup.action = action;
314 1223 : lookup.transport_name = (char *) transport_name;
315 :
316 1223 : return HT_FIND(clientmap, &client_history, &lookup);
317 : }
318 :
319 : /* Cleanup client entries older than the cutoff. Used for the OOM. Return the
320 : * number of bytes freed. If 0 is returned, nothing was freed. */
321 : static size_t
322 192 : oom_clean_client_entries(time_t cutoff)
323 : {
324 192 : size_t bytes = 0;
325 192 : clientmap_entry_t **ent, **ent_next;
326 :
327 347 : for (ent = HT_START(clientmap, &client_history); ent; ent = ent_next) {
328 155 : clientmap_entry_t *entry = *ent;
329 155 : if (entry->last_seen_in_minutes < (cutoff / 60)) {
330 1 : ent_next = HT_NEXT_RMV(clientmap, &client_history, ent);
331 1 : bytes += clientmap_entry_size(entry);
332 1 : clientmap_entry_free(entry);
333 : } else {
334 154 : ent_next = HT_NEXT(clientmap, &client_history, ent);
335 : }
336 : }
337 192 : return bytes;
338 : }
339 :
340 : /* Below this minimum lifetime, the OOM won't cleanup any entries. */
341 : #define GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF (4 * 60 * 60)
342 : /* The OOM moves the cutoff by that much every run. */
343 : #define GEOIP_CLIENT_CACHE_OOM_STEP (15 * 50)
344 :
345 : /* Cleanup the geoip client history cache called from the OOM handler. Return
346 : * the amount of bytes removed. This can return a value below or above
347 : * min_remove_bytes but will stop as oon as the min_remove_bytes has been
348 : * reached. */
349 : size_t
350 2 : geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes)
351 : {
352 2 : time_t k;
353 2 : size_t bytes_removed = 0;
354 :
355 : /* Our OOM handler called with 0 bytes to remove is a code flow error. */
356 2 : tor_assert(min_remove_bytes != 0);
357 :
358 : /* Set k to the initial cutoff of an entry. We then going to move it by step
359 : * to try to remove as much as we can. */
360 : k = WRITE_STATS_INTERVAL;
361 :
362 194 : do {
363 194 : time_t cutoff;
364 :
365 : /* If k has reached the minimum lifetime, we have to stop else we might
366 : * remove every single entries which would be pretty bad for the DoS
367 : * mitigation subsystem if by just filling the geoip cache, it was enough
368 : * to trigger the OOM and clean every single entries. */
369 194 : if (k <= GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF) {
370 : break;
371 : }
372 :
373 192 : cutoff = now - k;
374 192 : bytes_removed += oom_clean_client_entries(cutoff);
375 192 : k -= GEOIP_CLIENT_CACHE_OOM_STEP;
376 192 : } while (bytes_removed < min_remove_bytes);
377 :
378 2 : return bytes_removed;
379 : }
380 :
381 : /* Return the total size in bytes of the client history cache. */
382 : size_t
383 12 : geoip_client_cache_total_allocation(void)
384 : {
385 12 : return geoip_client_history_cache_size;
386 : }
387 :
388 : /** How many responses are we giving to clients requesting v3 network
389 : * statuses? */
390 : static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM];
391 :
392 : /** Note that we've rejected a client's request for a v3 network status
393 : * for reason <b>reason</b> at time <b>now</b>. */
394 : void
395 7 : geoip_note_ns_response(geoip_ns_response_t response)
396 : {
397 7 : static int arrays_initialized = 0;
398 7 : if (!get_options()->DirReqStatistics)
399 : return;
400 5 : if (!arrays_initialized) {
401 5 : memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
402 5 : arrays_initialized = 1;
403 : }
404 5 : tor_assert(response < GEOIP_NS_RESPONSE_NUM);
405 5 : ns_v3_responses[response]++;
406 : }
407 :
408 : /** Do not mention any country from which fewer than this number of IPs have
409 : * connected. This conceivably avoids reporting information that could
410 : * deanonymize users, though analysis is lacking. */
411 : #define MIN_IPS_TO_NOTE_COUNTRY 1
412 : /** Do not report any geoip data at all if we have fewer than this number of
413 : * IPs to report about. */
414 : #define MIN_IPS_TO_NOTE_ANYTHING 1
415 : /** When reporting geoip data about countries, round up to the nearest
416 : * multiple of this value. */
417 : #define IP_GRANULARITY 8
418 :
419 : /** Helper type: used to sort per-country totals by value. */
420 : typedef struct c_hist_t {
421 : char country[3]; /**< Two-letter country code. */
422 : unsigned total; /**< Total IP addresses seen in this country. */
423 : } c_hist_t;
424 :
425 : /** Sorting helper: return -1, 1, or 0 based on comparison of two
426 : * geoip_ipv4_entry_t. Sort in descending order of total, and then by country
427 : * code. */
428 : static int
429 5 : c_hist_compare_(const void **_a, const void **_b)
430 : {
431 5 : const c_hist_t *a = *_a, *b = *_b;
432 5 : if (a->total > b->total)
433 : return -1;
434 4 : else if (a->total < b->total)
435 : return 1;
436 : else
437 0 : return strcmp(a->country, b->country);
438 : }
439 :
440 : /** When there are incomplete directory requests at the end of a 24-hour
441 : * period, consider those requests running for longer than this timeout as
442 : * failed, the others as still running. */
443 : #define DIRREQ_TIMEOUT (10*60)
444 :
445 : /** Entry in a map from either chan->global_identifier for direct requests
446 : * or a unique circuit identifier for tunneled requests to request time,
447 : * response size, and completion time of a network status request. Used to
448 : * measure download times of requests to derive average client
449 : * bandwidths. */
450 : typedef struct dirreq_map_entry_t {
451 : HT_ENTRY(dirreq_map_entry_t) node;
452 : /** Unique identifier for this network status request; this is either the
453 : * chan->global_identifier of the dir channel (direct request) or a new
454 : * locally unique identifier of a circuit (tunneled request). This ID is
455 : * only unique among other direct or tunneled requests, respectively. */
456 : uint64_t dirreq_id;
457 : unsigned int state:3; /**< State of this directory request. */
458 : unsigned int type:1; /**< Is this a direct or a tunneled request? */
459 : unsigned int completed:1; /**< Is this request complete? */
460 : /** When did we receive the request and started sending the response? */
461 : struct timeval request_time;
462 : size_t response_size; /**< What is the size of the response in bytes? */
463 : struct timeval completion_time; /**< When did the request succeed? */
464 : } dirreq_map_entry_t;
465 :
466 : /** Map of all directory requests asking for v2 or v3 network statuses in
467 : * the current geoip-stats interval. Values are
468 : * of type *<b>dirreq_map_entry_t</b>. */
469 : static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map =
470 : HT_INITIALIZER();
471 :
472 : static int
473 0 : dirreq_map_ent_eq(const dirreq_map_entry_t *a,
474 : const dirreq_map_entry_t *b)
475 : {
476 0 : return a->dirreq_id == b->dirreq_id && a->type == b->type;
477 : }
478 :
479 : /* DOCDOC dirreq_map_ent_hash */
480 : static unsigned
481 2 : dirreq_map_ent_hash(const dirreq_map_entry_t *entry)
482 : {
483 2 : unsigned u = (unsigned) entry->dirreq_id;
484 2 : u += entry->type << 20;
485 2 : return u;
486 : }
487 :
488 758 : HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
489 : dirreq_map_ent_eq);
490 237 : HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
491 : dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_);
492 :
493 : /** Helper: Put <b>entry</b> into map of directory requests using
494 : * <b>type</b> and <b>dirreq_id</b> as key parts. If there is
495 : * already an entry for that key, print out a BUG warning and return. */
496 : static void
497 2 : dirreq_map_put_(dirreq_map_entry_t *entry, dirreq_type_t type,
498 : uint64_t dirreq_id)
499 : {
500 2 : dirreq_map_entry_t *old_ent;
501 2 : tor_assert(entry->type == type);
502 2 : tor_assert(entry->dirreq_id == dirreq_id);
503 :
504 : /* XXXX we could switch this to HT_INSERT some time, since it seems that
505 : * this bug doesn't happen. But since this function doesn't seem to be
506 : * critical-path, it's sane to leave it alone. */
507 2 : old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry);
508 2 : if (old_ent && old_ent != entry) {
509 0 : log_warn(LD_BUG, "Error when putting directory request into local "
510 : "map. There was already an entry for the same identifier.");
511 0 : return;
512 : }
513 : }
514 :
515 : /** Helper: Look up and return an entry in the map of directory requests
516 : * using <b>type</b> and <b>dirreq_id</b> as key parts. If there
517 : * is no such entry, return NULL. */
518 : static dirreq_map_entry_t *
519 0 : dirreq_map_get_(dirreq_type_t type, uint64_t dirreq_id)
520 : {
521 0 : dirreq_map_entry_t lookup;
522 0 : lookup.type = type;
523 0 : lookup.dirreq_id = dirreq_id;
524 0 : return HT_FIND(dirreqmap, &dirreq_map, &lookup);
525 : }
526 :
527 : /** Note that an either direct or tunneled (see <b>type</b>) directory
528 : * request for a v3 network status with unique ID <b>dirreq_id</b> of size
529 : * <b>response_size</b> has started. */
530 : void
531 2 : geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
532 : dirreq_type_t type)
533 : {
534 2 : dirreq_map_entry_t *ent;
535 2 : if (!get_options()->DirReqStatistics)
536 : return;
537 2 : ent = tor_malloc_zero(sizeof(dirreq_map_entry_t));
538 2 : ent->dirreq_id = dirreq_id;
539 2 : tor_gettimeofday(&ent->request_time);
540 2 : ent->response_size = response_size;
541 2 : ent->type = type;
542 2 : dirreq_map_put_(ent, type, dirreq_id);
543 : }
544 :
545 : /** Change the state of the either direct or tunneled (see <b>type</b>)
546 : * directory request with <b>dirreq_id</b> to <b>new_state</b> and
547 : * possibly mark it as completed. If no entry can be found for the given
548 : * key parts (e.g., if this is a directory request that we are not
549 : * measuring, or one that was started in the previous measurement period),
550 : * or if the state cannot be advanced to <b>new_state</b>, do nothing. */
551 : void
552 0 : geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
553 : dirreq_state_t new_state)
554 : {
555 0 : dirreq_map_entry_t *ent;
556 0 : if (!get_options()->DirReqStatistics)
557 : return;
558 0 : ent = dirreq_map_get_(type, dirreq_id);
559 0 : if (!ent)
560 : return;
561 0 : if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS)
562 : return;
563 0 : if (new_state - 1 != ent->state)
564 : return;
565 0 : ent->state = new_state;
566 0 : if ((type == DIRREQ_DIRECT &&
567 0 : new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) ||
568 0 : (type == DIRREQ_TUNNELED &&
569 0 : new_state == DIRREQ_CHANNEL_BUFFER_FLUSHED)) {
570 0 : tor_gettimeofday(&ent->completion_time);
571 0 : ent->completed = 1;
572 : }
573 : }
574 :
575 : /** Return the bridge-ip-transports string that should be inserted in
576 : * our extra-info descriptor. Return NULL if the bridge-ip-transports
577 : * line should be empty. */
578 : char *
579 3 : geoip_get_transport_history(void)
580 : {
581 3 : unsigned granularity = IP_GRANULARITY;
582 : /** String hash table (name of transport) -> (number of users). */
583 3 : strmap_t *transport_counts = strmap_new();
584 :
585 : /** Smartlist that contains copies of the names of the transports
586 : that have been used. */
587 3 : smartlist_t *transports_used = smartlist_new();
588 :
589 : /* Special string to signify that no transport was used for this
590 : connection. Pluggable transport names can't have symbols in their
591 : names, so this string will never collide with a real transport. */
592 3 : static const char* no_transport_str = "<OR>";
593 :
594 3 : clientmap_entry_t **ent;
595 3 : smartlist_t *string_chunks = smartlist_new();
596 3 : char *the_string = NULL;
597 :
598 : /* If we haven't seen any clients yet, return NULL. */
599 3 : if (HT_EMPTY(&client_history))
600 1 : goto done;
601 :
602 : /** We do the following steps to form the transport history string:
603 : * a) Foreach client that uses a pluggable transport, we increase the
604 : * times that transport was used by one. If the client did not use
605 : * a transport, we increase the number of times someone connected
606 : * without obfuscation.
607 : * b) Foreach transport we observed, we write its transport history
608 : * string and push it to string_chunks. So, for example, if we've
609 : * seen 665 obfs2 clients, we write "obfs2=665".
610 : * c) We concatenate string_chunks to form the final string.
611 : */
612 :
613 2 : log_debug(LD_GENERAL,"Starting iteration for transport history. %d clients.",
614 : HT_SIZE(&client_history));
615 :
616 : /* Loop through all clients. */
617 191 : HT_FOREACH(ent, clientmap, &client_history) {
618 189 : uintptr_t val;
619 189 : void *ptr;
620 189 : const char *transport_name = (*ent)->transport_name;
621 189 : if (!transport_name)
622 24 : transport_name = no_transport_str;
623 :
624 : /* Increase the count for this transport name. */
625 189 : ptr = strmap_get(transport_counts, transport_name);
626 189 : val = (uintptr_t)ptr;
627 189 : val++;
628 189 : ptr = (void*)val;
629 189 : strmap_set(transport_counts, transport_name, ptr);
630 :
631 : /* If it's the first time we see this transport, note it. */
632 189 : if (val == 1)
633 9 : smartlist_add_strdup(transports_used, transport_name);
634 :
635 189 : log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. "
636 : "I've now seen %d clients.",
637 : safe_str_client(fmt_addr(&(*ent)->addr)),
638 : transport_name ? transport_name : "<no transport>",
639 : (int)val);
640 : }
641 :
642 : /* Sort the transport names (helps with unit testing). */
643 2 : smartlist_sort_strings(transports_used);
644 :
645 : /* Loop through all seen transports. */
646 11 : SMARTLIST_FOREACH_BEGIN(transports_used, const char *, transport_name) {
647 9 : void *transport_count_ptr = strmap_get(transport_counts, transport_name);
648 9 : uintptr_t transport_count = (uintptr_t) transport_count_ptr;
649 :
650 9 : log_debug(LD_GENERAL, "We got %"PRIu64" clients with transport '%s'.",
651 : ((uint64_t)transport_count), transport_name);
652 :
653 9 : smartlist_add_asprintf(string_chunks, "%s=%"PRIu64,
654 : transport_name,
655 : (round_uint64_to_next_multiple_of(
656 : (uint64_t)transport_count,
657 : granularity)));
658 9 : } SMARTLIST_FOREACH_END(transport_name);
659 :
660 2 : the_string = smartlist_join_strings(string_chunks, ",", 0, NULL);
661 :
662 2 : log_debug(LD_GENERAL, "Final bridge-ip-transports string: '%s'", the_string);
663 :
664 3 : done:
665 3 : strmap_free(transport_counts, NULL);
666 12 : SMARTLIST_FOREACH(transports_used, char *, s, tor_free(s));
667 3 : smartlist_free(transports_used);
668 12 : SMARTLIST_FOREACH(string_chunks, char *, s, tor_free(s));
669 3 : smartlist_free(string_chunks);
670 :
671 3 : return the_string;
672 : }
673 :
674 : /** Return a newly allocated comma-separated string containing statistics
675 : * on network status downloads. The string contains the number of completed
676 : * requests, timeouts, and still running requests as well as the download
677 : * times by deciles and quartiles. Return NULL if we have not observed
678 : * requests for long enough. */
679 : static char *
680 16 : geoip_get_dirreq_history(dirreq_type_t type)
681 : {
682 16 : char *result = NULL;
683 16 : buf_t *buf = NULL;
684 16 : smartlist_t *dirreq_completed = NULL;
685 16 : uint32_t complete = 0, timeouts = 0, running = 0;
686 16 : dirreq_map_entry_t **ptr, **next;
687 16 : struct timeval now;
688 :
689 16 : tor_gettimeofday(&now);
690 16 : dirreq_completed = smartlist_new();
691 19 : for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) {
692 3 : dirreq_map_entry_t *ent = *ptr;
693 3 : if (ent->type != type) {
694 1 : next = HT_NEXT(dirreqmap, &dirreq_map, ptr);
695 1 : continue;
696 : } else {
697 2 : if (ent->completed) {
698 0 : smartlist_add(dirreq_completed, ent);
699 0 : complete++;
700 0 : next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
701 : } else {
702 2 : if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT)
703 0 : timeouts++;
704 : else
705 2 : running++;
706 2 : next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
707 2 : tor_free(ent);
708 : }
709 : }
710 : }
711 : #define DIR_REQ_GRANULARITY 4
712 16 : complete = round_uint32_to_next_multiple_of(complete,
713 : DIR_REQ_GRANULARITY);
714 16 : timeouts = round_uint32_to_next_multiple_of(timeouts,
715 : DIR_REQ_GRANULARITY);
716 16 : running = round_uint32_to_next_multiple_of(running,
717 : DIR_REQ_GRANULARITY);
718 16 : buf = buf_new_with_capacity(1024);
719 16 : buf_add_printf(buf, "complete=%u,timeout=%u,"
720 : "running=%u", complete, timeouts, running);
721 :
722 : #define MIN_DIR_REQ_RESPONSES 16
723 16 : if (complete >= MIN_DIR_REQ_RESPONSES) {
724 0 : uint32_t *dltimes;
725 : /* We may have rounded 'completed' up. Here we want to use the
726 : * real value. */
727 0 : complete = smartlist_len(dirreq_completed);
728 0 : dltimes = tor_calloc(complete, sizeof(uint32_t));
729 0 : SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) {
730 0 : uint32_t bytes_per_second;
731 0 : uint32_t time_diff_ = (uint32_t) tv_mdiff(&ent->request_time,
732 0 : &ent->completion_time);
733 0 : if (time_diff_ == 0)
734 : time_diff_ = 1; /* Avoid DIV/0; "instant" answers are impossible
735 : * by law of nature or something, but a millisecond
736 : * is a bit greater than "instantly" */
737 0 : bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff_);
738 0 : dltimes[ent_sl_idx] = bytes_per_second;
739 0 : } SMARTLIST_FOREACH_END(ent);
740 0 : median_uint32(dltimes, complete); /* sorts as a side effect. */
741 0 : buf_add_printf(buf,
742 : ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u,"
743 : "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u",
744 : dltimes[0],
745 0 : dltimes[1*complete/10-1],
746 0 : dltimes[2*complete/10-1],
747 0 : dltimes[1*complete/4-1],
748 0 : dltimes[3*complete/10-1],
749 0 : dltimes[4*complete/10-1],
750 0 : dltimes[5*complete/10-1],
751 0 : dltimes[6*complete/10-1],
752 0 : dltimes[7*complete/10-1],
753 0 : dltimes[3*complete/4-1],
754 0 : dltimes[8*complete/10-1],
755 0 : dltimes[9*complete/10-1],
756 0 : dltimes[complete-1]);
757 0 : tor_free(dltimes);
758 : }
759 :
760 16 : result = buf_extract(buf, NULL);
761 :
762 16 : SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent,
763 : tor_free(ent));
764 16 : smartlist_free(dirreq_completed);
765 16 : buf_free(buf);
766 16 : return result;
767 : }
768 :
769 : /** Store a newly allocated comma-separated string in
770 : * *<a>country_str</a> containing entries for all the countries from
771 : * which we've seen enough clients connect as a bridge, directory
772 : * server, or entry guard. The entry format is cc=num where num is the
773 : * number of IPs we've seen connecting from that country, and cc is a
774 : * lowercased country code. *<a>country_str</a> is set to NULL if
775 : * we're not ready to export per country data yet.
776 : *
777 : * Store a newly allocated comma-separated string in <a>ipver_str</a>
778 : * containing entries for clients connecting over IPv4 and IPv6. The
779 : * format is family=num where num is the number of IPs we've seen
780 : * connecting over that protocol family, and family is 'v4' or 'v6'.
781 : *
782 : * Return 0 on success and -1 if we're missing geoip data. */
783 : int
784 13 : geoip_get_client_history(geoip_client_action_t action,
785 : char **country_str, char **ipver_str)
786 : {
787 13 : unsigned granularity = IP_GRANULARITY;
788 13 : smartlist_t *entries = NULL;
789 13 : int n_countries = geoip_get_n_countries();
790 13 : int i;
791 13 : clientmap_entry_t **cm_ent;
792 13 : unsigned *counts = NULL;
793 13 : unsigned total = 0;
794 13 : unsigned ipv4_count = 0, ipv6_count = 0;
795 :
796 13 : if (!geoip_is_loaded(AF_INET) && !geoip_is_loaded(AF_INET6))
797 : return -1;
798 :
799 11 : counts = tor_calloc(n_countries, sizeof(unsigned));
800 83 : HT_FOREACH(cm_ent, clientmap, &client_history) {
801 72 : int country;
802 72 : if ((*cm_ent)->action != (int)action)
803 0 : continue;
804 72 : country = geoip_get_country_by_addr(&(*cm_ent)->addr);
805 72 : if (country < 0)
806 : country = 0; /** unresolved requests are stored at index 0. */
807 72 : tor_assert(0 <= country && country < n_countries);
808 72 : ++counts[country];
809 72 : ++total;
810 72 : switch (tor_addr_family(&(*cm_ent)->addr)) {
811 40 : case AF_INET:
812 40 : ipv4_count++;
813 40 : break;
814 32 : case AF_INET6:
815 32 : ipv6_count++;
816 32 : break;
817 : }
818 : }
819 11 : if (ipver_str) {
820 3 : smartlist_t *chunks = smartlist_new();
821 3 : smartlist_add_asprintf(chunks, "v4=%u",
822 : round_to_next_multiple_of(ipv4_count, granularity));
823 3 : smartlist_add_asprintf(chunks, "v6=%u",
824 : round_to_next_multiple_of(ipv6_count, granularity));
825 3 : *ipver_str = smartlist_join_strings(chunks, ",", 0, NULL);
826 9 : SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
827 3 : smartlist_free(chunks);
828 : }
829 :
830 : /* Don't record per country data if we haven't seen enough IPs. */
831 11 : if (total < MIN_IPS_TO_NOTE_ANYTHING) {
832 5 : tor_free(counts);
833 5 : if (country_str)
834 5 : *country_str = NULL;
835 5 : return 0;
836 : }
837 :
838 : /* Make a list of c_hist_t */
839 6 : entries = smartlist_new();
840 34 : for (i = 0; i < n_countries; ++i) {
841 22 : unsigned c = counts[i];
842 22 : const char *countrycode;
843 22 : c_hist_t *ent;
844 : /* Only report a country if it has a minimum number of IPs. */
845 22 : if (c >= MIN_IPS_TO_NOTE_COUNTRY) {
846 10 : c = round_to_next_multiple_of(c, granularity);
847 10 : countrycode = geoip_get_country_name(i);
848 10 : ent = tor_malloc(sizeof(c_hist_t));
849 10 : strlcpy(ent->country, countrycode, sizeof(ent->country));
850 10 : ent->total = c;
851 10 : smartlist_add(entries, ent);
852 : }
853 : }
854 : /* Sort entries. Note that we must do this _AFTER_ rounding, or else
855 : * the sort order could leak info. */
856 6 : smartlist_sort(entries, c_hist_compare_);
857 :
858 6 : if (country_str) {
859 6 : smartlist_t *chunks = smartlist_new();
860 16 : SMARTLIST_FOREACH(entries, c_hist_t *, ch, {
861 : smartlist_add_asprintf(chunks, "%s=%u", ch->country, ch->total);
862 : });
863 6 : *country_str = smartlist_join_strings(chunks, ",", 0, NULL);
864 16 : SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
865 6 : smartlist_free(chunks);
866 : }
867 :
868 16 : SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c));
869 6 : smartlist_free(entries);
870 6 : tor_free(counts);
871 :
872 6 : return 0;
873 : }
874 :
875 : /** Return a newly allocated string holding the per-country request history
876 : * for v3 network statuses in a format suitable for an extra-info document,
877 : * or NULL on failure. */
878 : char *
879 9 : geoip_get_request_history(void)
880 : {
881 9 : smartlist_t *entries, *strings;
882 9 : char *result;
883 9 : unsigned granularity = IP_GRANULARITY;
884 :
885 9 : entries = smartlist_new();
886 33 : SMARTLIST_FOREACH_BEGIN(geoip_get_countries(), const geoip_country_t *, c) {
887 24 : uint32_t tot = 0;
888 24 : c_hist_t *ent;
889 24 : if ((size_t)c_sl_idx < n_v3_ns_requests_len)
890 20 : tot = n_v3_ns_requests[c_sl_idx];
891 : else
892 : tot = 0;
893 20 : if (!tot)
894 21 : continue;
895 3 : ent = tor_malloc_zero(sizeof(c_hist_t));
896 3 : strlcpy(ent->country, c->countrycode, sizeof(ent->country));
897 3 : ent->total = round_to_next_multiple_of(tot, granularity);
898 3 : smartlist_add(entries, ent);
899 24 : } SMARTLIST_FOREACH_END(c);
900 9 : smartlist_sort(entries, c_hist_compare_);
901 :
902 9 : strings = smartlist_new();
903 12 : SMARTLIST_FOREACH(entries, c_hist_t *, ent, {
904 : smartlist_add_asprintf(strings, "%s=%u", ent->country, ent->total);
905 : });
906 9 : result = smartlist_join_strings(strings, ",", 0, NULL);
907 12 : SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp));
908 12 : SMARTLIST_FOREACH(entries, c_hist_t *, ent, tor_free(ent));
909 9 : smartlist_free(strings);
910 9 : smartlist_free(entries);
911 9 : return result;
912 : }
913 :
914 : /** Start time of directory request stats or 0 if we're not collecting
915 : * directory request statistics. */
916 : static time_t start_of_dirreq_stats_interval;
917 :
918 : /** Initialize directory request stats. */
919 : void
920 6 : geoip_dirreq_stats_init(time_t now)
921 : {
922 6 : start_of_dirreq_stats_interval = now;
923 6 : }
924 :
925 : /** Reset counters for dirreq stats. */
926 : void
927 3 : geoip_reset_dirreq_stats(time_t now)
928 : {
929 3 : memset(n_v3_ns_requests, 0,
930 : n_v3_ns_requests_len * sizeof(uint32_t));
931 : {
932 3 : clientmap_entry_t **ent, **next, *this;
933 6 : for (ent = HT_START(clientmap, &client_history); ent != NULL;
934 : ent = next) {
935 3 : if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS) {
936 3 : this = *ent;
937 3 : next = HT_NEXT_RMV(clientmap, &client_history, ent);
938 3 : clientmap_entry_free(this);
939 : } else {
940 0 : next = HT_NEXT(clientmap, &client_history, ent);
941 : }
942 : }
943 : }
944 3 : memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
945 : {
946 3 : dirreq_map_entry_t **ent, **next, *this;
947 3 : for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
948 0 : this = *ent;
949 0 : next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
950 0 : tor_free(this);
951 : }
952 : }
953 3 : start_of_dirreq_stats_interval = now;
954 3 : }
955 :
956 : /** Stop collecting directory request stats in a way that we can re-start
957 : * doing so in geoip_dirreq_stats_init(). */
958 : void
959 2 : geoip_dirreq_stats_term(void)
960 : {
961 2 : geoip_reset_dirreq_stats(0);
962 2 : }
963 :
964 : /** Return a newly allocated string containing the dirreq statistics
965 : * until <b>now</b>, or NULL if we're not collecting dirreq stats. Caller
966 : * must ensure start_of_dirreq_stats_interval is in the past. */
967 : char *
968 10 : geoip_format_dirreq_stats(time_t now)
969 : {
970 10 : char t[ISO_TIME_LEN+1];
971 10 : int i;
972 10 : char *v3_ips_string = NULL, *v3_reqs_string = NULL,
973 10 : *v3_direct_dl_string = NULL, *v3_tunneled_dl_string = NULL;
974 10 : char *result = NULL;
975 :
976 10 : if (!start_of_dirreq_stats_interval)
977 : return NULL; /* Not initialized. */
978 :
979 8 : tor_assert(now >= start_of_dirreq_stats_interval);
980 :
981 8 : format_iso_time(t, now);
982 8 : geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS, &v3_ips_string, NULL);
983 8 : v3_reqs_string = geoip_get_request_history();
984 :
985 : #define RESPONSE_GRANULARITY 8
986 64 : for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) {
987 48 : ns_v3_responses[i] = round_uint32_to_next_multiple_of(
988 : ns_v3_responses[i], RESPONSE_GRANULARITY);
989 : }
990 : #undef RESPONSE_GRANULARITY
991 :
992 8 : v3_direct_dl_string = geoip_get_dirreq_history(DIRREQ_DIRECT);
993 8 : v3_tunneled_dl_string = geoip_get_dirreq_history(DIRREQ_TUNNELED);
994 :
995 : /* Put everything together into a single string. */
996 32 : tor_asprintf(&result, "dirreq-stats-end %s (%d s)\n"
997 : "dirreq-v3-ips %s\n"
998 : "dirreq-v3-reqs %s\n"
999 : "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u,"
1000 : "not-found=%u,not-modified=%u,busy=%u\n"
1001 : "dirreq-v3-direct-dl %s\n"
1002 : "dirreq-v3-tunneled-dl %s\n",
1003 : t,
1004 : (unsigned) (now - start_of_dirreq_stats_interval),
1005 8 : v3_ips_string ? v3_ips_string : "",
1006 : v3_reqs_string ? v3_reqs_string : "",
1007 : ns_v3_responses[GEOIP_SUCCESS],
1008 : ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS],
1009 : ns_v3_responses[GEOIP_REJECT_UNAVAILABLE],
1010 : ns_v3_responses[GEOIP_REJECT_NOT_FOUND],
1011 : ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED],
1012 : ns_v3_responses[GEOIP_REJECT_BUSY],
1013 : v3_direct_dl_string ? v3_direct_dl_string : "",
1014 : v3_tunneled_dl_string ? v3_tunneled_dl_string : "");
1015 :
1016 : /* Free partial strings. */
1017 8 : tor_free(v3_ips_string);
1018 8 : tor_free(v3_reqs_string);
1019 8 : tor_free(v3_direct_dl_string);
1020 8 : tor_free(v3_tunneled_dl_string);
1021 :
1022 8 : return result;
1023 : }
1024 :
1025 : /** If 24 hours have passed since the beginning of the current dirreq
1026 : * stats period, write dirreq stats to $DATADIR/stats/dirreq-stats
1027 : * (possibly overwriting an existing file) and reset counters. Return
1028 : * when we would next want to write dirreq stats or 0 if we never want to
1029 : * write. */
1030 : time_t
1031 0 : geoip_dirreq_stats_write(time_t now)
1032 : {
1033 0 : char *str = NULL;
1034 :
1035 0 : if (!start_of_dirreq_stats_interval)
1036 : return 0; /* Not initialized. */
1037 0 : if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now)
1038 0 : goto done; /* Not ready to write. */
1039 :
1040 : /* Discard all items in the client history that are too old. */
1041 0 : geoip_remove_old_clients(start_of_dirreq_stats_interval);
1042 :
1043 : /* Generate history string .*/
1044 0 : str = geoip_format_dirreq_stats(now);
1045 0 : if (! str)
1046 0 : goto done;
1047 :
1048 : /* Write dirreq-stats string to disk. */
1049 0 : if (!check_or_create_data_subdir("stats")) {
1050 0 : write_to_data_subdir("stats", "dirreq-stats", str, "dirreq statistics");
1051 : /* Reset measurement interval start. */
1052 0 : geoip_reset_dirreq_stats(now);
1053 : }
1054 :
1055 0 : done:
1056 0 : tor_free(str);
1057 0 : return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL;
1058 : }
1059 :
1060 : /** Start time of bridge stats or 0 if we're not collecting bridge
1061 : * statistics. */
1062 : static time_t start_of_bridge_stats_interval;
1063 :
1064 : /** Initialize bridge stats. */
1065 : void
1066 1 : geoip_bridge_stats_init(time_t now)
1067 : {
1068 1 : start_of_bridge_stats_interval = now;
1069 1 : }
1070 :
1071 : /** Stop collecting bridge stats in a way that we can re-start doing so in
1072 : * geoip_bridge_stats_init(). */
1073 : void
1074 2 : geoip_bridge_stats_term(void)
1075 : {
1076 2 : client_history_clear();
1077 2 : start_of_bridge_stats_interval = 0;
1078 2 : }
1079 :
1080 : /** Validate a bridge statistics string as it would be written to a
1081 : * current extra-info descriptor. Return 1 if the string is valid and
1082 : * recent enough, or 0 otherwise. */
1083 : static int
1084 0 : validate_bridge_stats(const char *stats_str, time_t now)
1085 : {
1086 0 : char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1],
1087 : *eos;
1088 :
1089 0 : const char *BRIDGE_STATS_END = "bridge-stats-end ";
1090 0 : const char *BRIDGE_IPS = "bridge-ips ";
1091 0 : const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n";
1092 0 : const char *BRIDGE_TRANSPORTS = "bridge-ip-transports ";
1093 0 : const char *BRIDGE_TRANSPORTS_EMPTY_LINE = "bridge-ip-transports\n";
1094 0 : const char *tmp;
1095 0 : time_t stats_end_time;
1096 0 : int seconds;
1097 0 : tor_assert(stats_str);
1098 :
1099 : /* Parse timestamp and number of seconds from
1100 : "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */
1101 0 : tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END);
1102 0 : if (!tmp)
1103 : return 0;
1104 0 : tmp += strlen(BRIDGE_STATS_END);
1105 :
1106 0 : if (strlen(tmp) < ISO_TIME_LEN + 6)
1107 : return 0;
1108 0 : strlcpy(stats_end_str, tmp, sizeof(stats_end_str));
1109 0 : if (parse_iso_time(stats_end_str, &stats_end_time) < 0)
1110 : return 0;
1111 0 : if (stats_end_time < now - (25*60*60) ||
1112 0 : stats_end_time > now + (1*60*60))
1113 : return 0;
1114 0 : seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10);
1115 0 : if (!eos || seconds < 23*60*60)
1116 : return 0;
1117 0 : format_iso_time(stats_start_str, stats_end_time - seconds);
1118 :
1119 : /* Parse: "bridge-ips CC=N,CC=N,..." */
1120 0 : tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS);
1121 0 : if (!tmp) {
1122 : /* Look if there is an empty "bridge-ips" line */
1123 0 : tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE);
1124 0 : if (!tmp)
1125 : return 0;
1126 : }
1127 :
1128 : /* Parse: "bridge-ip-transports PT=N,PT=N,..." */
1129 0 : tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS);
1130 0 : if (!tmp) {
1131 : /* Look if there is an empty "bridge-ip-transports" line */
1132 0 : tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS_EMPTY_LINE);
1133 0 : if (!tmp)
1134 0 : return 0;
1135 : }
1136 :
1137 : return 1;
1138 : }
1139 :
1140 : /** Most recent bridge statistics formatted to be written to extra-info
1141 : * descriptors. */
1142 : static char *bridge_stats_extrainfo = NULL;
1143 :
1144 : /** Return a newly allocated string holding our bridge usage stats by country
1145 : * in a format suitable for inclusion in an extrainfo document. Return NULL on
1146 : * failure. */
1147 : char *
1148 3 : geoip_format_bridge_stats(time_t now)
1149 : {
1150 3 : char *out = NULL;
1151 3 : char *country_data = NULL, *ipver_data = NULL, *transport_data = NULL;
1152 3 : long duration = now - start_of_bridge_stats_interval;
1153 3 : char written[ISO_TIME_LEN+1];
1154 :
1155 3 : if (duration < 0)
1156 : return NULL;
1157 3 : if (!start_of_bridge_stats_interval)
1158 : return NULL; /* Not initialized. */
1159 :
1160 1 : format_iso_time(written, now);
1161 1 : geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
1162 1 : transport_data = geoip_get_transport_history();
1163 :
1164 2 : tor_asprintf(&out,
1165 : "bridge-stats-end %s (%ld s)\n"
1166 : "bridge-ips %s\n"
1167 : "bridge-ip-versions %s\n"
1168 : "bridge-ip-transports %s\n",
1169 : written, duration,
1170 1 : country_data ? country_data : "",
1171 1 : ipver_data ? ipver_data : "",
1172 : transport_data ? transport_data : "");
1173 1 : tor_free(country_data);
1174 1 : tor_free(ipver_data);
1175 1 : tor_free(transport_data);
1176 :
1177 1 : return out;
1178 : }
1179 :
1180 : /** Return a newly allocated string holding our bridge usage stats by country
1181 : * in a format suitable for the answer to a controller request. Return NULL on
1182 : * failure. */
1183 : static char *
1184 0 : format_bridge_stats_controller(time_t now)
1185 : {
1186 0 : char *out = NULL, *country_data = NULL, *ipver_data = NULL;
1187 0 : char started[ISO_TIME_LEN+1];
1188 0 : (void) now;
1189 :
1190 0 : format_iso_time(started, start_of_bridge_stats_interval);
1191 0 : geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
1192 :
1193 0 : tor_asprintf(&out,
1194 : "TimeStarted=\"%s\" CountrySummary=%s IPVersions=%s",
1195 : started,
1196 0 : country_data ? country_data : "",
1197 0 : ipver_data ? ipver_data : "");
1198 0 : tor_free(country_data);
1199 0 : tor_free(ipver_data);
1200 0 : return out;
1201 : }
1202 :
1203 : /** Return a newly allocated string holding our bridge usage stats by
1204 : * country in a format suitable for inclusion in our heartbeat
1205 : * message. Return NULL on failure. */
1206 : char *
1207 0 : format_client_stats_heartbeat(time_t now)
1208 : {
1209 0 : const int n_seconds = get_options()->HeartbeatPeriod;
1210 0 : char *out = NULL;
1211 0 : int n_clients = 0;
1212 0 : clientmap_entry_t **ent;
1213 0 : unsigned cutoff = (unsigned)( (now-n_seconds)/60 );
1214 :
1215 0 : if (!start_of_bridge_stats_interval)
1216 : return NULL; /* Not initialized. */
1217 :
1218 : /* count unique IPs */
1219 0 : HT_FOREACH(ent, clientmap, &client_history) {
1220 : /* only count directly connecting clients */
1221 0 : if ((*ent)->action != GEOIP_CLIENT_CONNECT)
1222 0 : continue;
1223 0 : if ((*ent)->last_seen_in_minutes < cutoff)
1224 0 : continue;
1225 0 : n_clients++;
1226 : }
1227 :
1228 0 : tor_asprintf(&out, "Heartbeat: "
1229 : "Since last heartbeat message, I have seen %d unique clients.",
1230 : n_clients);
1231 :
1232 0 : return out;
1233 : }
1234 :
1235 : /** Write bridge statistics to $DATADIR/stats/bridge-stats and return
1236 : * when we should next try to write statistics. */
1237 : time_t
1238 0 : geoip_bridge_stats_write(time_t now)
1239 : {
1240 0 : char *val = NULL;
1241 :
1242 : /* Check if 24 hours have passed since starting measurements. */
1243 0 : if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL)
1244 0 : return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
1245 :
1246 : /* Discard all items in the client history that are too old. */
1247 0 : geoip_remove_old_clients(start_of_bridge_stats_interval);
1248 :
1249 : /* Generate formatted string */
1250 0 : val = geoip_format_bridge_stats(now);
1251 0 : if (val == NULL)
1252 0 : goto done;
1253 :
1254 : /* Update the stored value. */
1255 0 : tor_free(bridge_stats_extrainfo);
1256 0 : bridge_stats_extrainfo = val;
1257 0 : start_of_bridge_stats_interval = now;
1258 :
1259 : /* Write it to disk. */
1260 0 : if (!check_or_create_data_subdir("stats")) {
1261 0 : write_to_data_subdir("stats", "bridge-stats",
1262 : bridge_stats_extrainfo, "bridge statistics");
1263 :
1264 : /* Tell the controller, "hey, there are clients!" */
1265 : {
1266 0 : char *controller_str = format_bridge_stats_controller(now);
1267 0 : if (controller_str)
1268 0 : control_event_clients_seen(controller_str);
1269 0 : tor_free(controller_str);
1270 : }
1271 : }
1272 :
1273 0 : done:
1274 0 : return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
1275 : }
1276 :
1277 : /** Try to load the most recent bridge statistics from disk, unless we
1278 : * have finished a measurement interval lately, and check whether they
1279 : * are still recent enough. */
1280 : static void
1281 3 : load_bridge_stats(time_t now)
1282 : {
1283 3 : char *fname, *contents;
1284 3 : if (bridge_stats_extrainfo)
1285 3 : return;
1286 :
1287 3 : fname = get_datadir_fname2("stats", "bridge-stats");
1288 3 : contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
1289 3 : if (contents && validate_bridge_stats(contents, now)) {
1290 0 : bridge_stats_extrainfo = contents;
1291 : } else {
1292 3 : tor_free(contents);
1293 : }
1294 :
1295 3 : tor_free(fname);
1296 : }
1297 :
1298 : /** Return most recent bridge statistics for inclusion in extra-info
1299 : * descriptors, or NULL if we don't have recent bridge statistics. */
1300 : const char *
1301 3 : geoip_get_bridge_stats_extrainfo(time_t now)
1302 : {
1303 3 : load_bridge_stats(now);
1304 3 : return bridge_stats_extrainfo;
1305 : }
1306 :
1307 : /** Return a new string containing the recent bridge statistics to be returned
1308 : * to controller clients, or NULL if we don't have any bridge statistics. */
1309 : char *
1310 0 : geoip_get_bridge_stats_controller(time_t now)
1311 : {
1312 0 : return format_bridge_stats_controller(now);
1313 : }
1314 :
1315 : /** Start time of entry stats or 0 if we're not collecting entry
1316 : * statistics. */
1317 : static time_t start_of_entry_stats_interval;
1318 :
1319 : /** Initialize entry stats. */
1320 : void
1321 4 : geoip_entry_stats_init(time_t now)
1322 : {
1323 4 : start_of_entry_stats_interval = now;
1324 4 : }
1325 :
1326 : /** Reset counters for entry stats. */
1327 : void
1328 4 : geoip_reset_entry_stats(time_t now)
1329 : {
1330 4 : client_history_clear();
1331 4 : start_of_entry_stats_interval = now;
1332 4 : }
1333 :
1334 : /** Stop collecting entry stats in a way that we can re-start doing so in
1335 : * geoip_entry_stats_init(). */
1336 : void
1337 3 : geoip_entry_stats_term(void)
1338 : {
1339 3 : geoip_reset_entry_stats(0);
1340 3 : }
1341 :
1342 : /** Return a newly allocated string containing the entry statistics
1343 : * until <b>now</b>, or NULL if we're not collecting entry stats. Caller
1344 : * must ensure start_of_entry_stats_interval lies in the past. */
1345 : char *
1346 4 : geoip_format_entry_stats(time_t now)
1347 : {
1348 4 : char t[ISO_TIME_LEN+1];
1349 4 : char *data = NULL;
1350 4 : char *result;
1351 :
1352 4 : if (!start_of_entry_stats_interval)
1353 : return NULL; /* Not initialized. */
1354 :
1355 2 : tor_assert(now >= start_of_entry_stats_interval);
1356 :
1357 2 : geoip_get_client_history(GEOIP_CLIENT_CONNECT, &data, NULL);
1358 2 : format_iso_time(t, now);
1359 2 : tor_asprintf(&result,
1360 : "entry-stats-end %s (%u s)\n"
1361 : "entry-ips %s\n",
1362 : t, (unsigned) (now - start_of_entry_stats_interval),
1363 2 : data ? data : "");
1364 2 : tor_free(data);
1365 2 : return result;
1366 : }
1367 :
1368 : /** If 24 hours have passed since the beginning of the current entry stats
1369 : * period, write entry stats to $DATADIR/stats/entry-stats (possibly
1370 : * overwriting an existing file) and reset counters. Return when we would
1371 : * next want to write entry stats or 0 if we never want to write. */
1372 : time_t
1373 0 : geoip_entry_stats_write(time_t now)
1374 : {
1375 0 : char *str = NULL;
1376 :
1377 0 : if (!start_of_entry_stats_interval)
1378 : return 0; /* Not initialized. */
1379 0 : if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now)
1380 0 : goto done; /* Not ready to write. */
1381 :
1382 : /* Discard all items in the client history that are too old. */
1383 0 : geoip_remove_old_clients(start_of_entry_stats_interval);
1384 :
1385 : /* Generate history string .*/
1386 0 : str = geoip_format_entry_stats(now);
1387 :
1388 : /* Write entry-stats string to disk. */
1389 0 : if (!check_or_create_data_subdir("stats")) {
1390 0 : write_to_data_subdir("stats", "entry-stats", str, "entry statistics");
1391 :
1392 : /* Reset measurement interval start. */
1393 0 : geoip_reset_entry_stats(now);
1394 : }
1395 :
1396 0 : done:
1397 0 : tor_free(str);
1398 0 : return start_of_entry_stats_interval + WRITE_STATS_INTERVAL;
1399 : }
1400 :
1401 : /** Release all storage held in this file. */
1402 : void
1403 235 : geoip_stats_free_all(void)
1404 : {
1405 : {
1406 235 : clientmap_entry_t **ent, **next, *this;
1407 235 : for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) {
1408 0 : this = *ent;
1409 0 : next = HT_NEXT_RMV(clientmap, &client_history, ent);
1410 0 : clientmap_entry_free(this);
1411 : }
1412 235 : HT_CLEAR(clientmap, &client_history);
1413 : }
1414 : {
1415 235 : dirreq_map_entry_t **ent, **next, *this;
1416 235 : for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
1417 0 : this = *ent;
1418 0 : next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
1419 0 : tor_free(this);
1420 : }
1421 235 : HT_CLEAR(dirreqmap, &dirreq_map);
1422 : }
1423 :
1424 235 : tor_free(bridge_stats_extrainfo);
1425 235 : tor_free(n_v3_ns_requests);
1426 235 : }
|
