37 #include "core/or/dos.h"
46 static size_t n_v3_ns_requests_len = 0;
49 static uint32_t *n_v3_ns_requests;
53 static size_t geoip_client_history_cache_size;
58 geoip_increment_client_history_cache_size(
size_t bytes)
61 IF_BUG_ONCE(geoip_client_history_cache_size > (SIZE_MAX - bytes)) {
62 geoip_client_history_cache_size = SIZE_MAX;
65 geoip_client_history_cache_size += bytes;
71 geoip_decrement_client_history_cache_size(
size_t bytes)
76 IF_BUG_ONCE(geoip_client_history_cache_size < bytes) {
77 geoip_client_history_cache_size = 0;
80 geoip_client_history_cache_size -= bytes;
85 increment_v3_ns_request(
country_t country)
90 if ((
size_t)country >= n_v3_ns_requests_len) {
93 if (n_v3_ns_requests_len == 0)
96 new_len = n_v3_ns_requests_len * 2;
97 if (new_len <= (
size_t)country)
98 new_len = ((size_t)country)+1;
99 n_v3_ns_requests = tor_reallocarray(n_v3_ns_requests, new_len,
101 memset(n_v3_ns_requests + n_v3_ns_requests_len, 0,
102 sizeof(uint32_t)*(new_len - n_v3_ns_requests_len));
103 n_v3_ns_requests_len = new_len;
106 n_v3_ns_requests[country] += 1;
120 #define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu
127 static inline unsigned
132 if (a->transport_name)
133 h += (unsigned) siphash24g(a->transport_name, strlen(a->transport_name));
141 if (
strcmp_opt(a->transport_name, b->transport_name))
145 a->action == b->action;
149 clientmap_entries_eq);
153 #define clientmap_entry_free(ent) \
154 FREE_AND_NULL(clientmap_entry_t, clientmap_entry_free_, ent)
162 (ent->transport_name ? strlen(ent->transport_name) : 0));
174 dos_geoip_entry_about_to_free(ent);
175 geoip_decrement_client_history_cache_size(clientmap_entry_size(ent));
185 const char *transport_name)
194 entry->action = action;
196 if (transport_name) {
197 entry->transport_name = tor_strdup(transport_name);
200 dos_geoip_entry_init(entry);
203 geoip_increment_client_history_cache_size(clientmap_entry_size(entry));
210 client_history_clear(
void)
213 for (ent = HT_START(clientmap, &client_history); ent != NULL;
217 next = HT_NEXT_RMV(clientmap, &client_history, ent);
218 clientmap_entry_free(
this);
220 next = HT_NEXT(clientmap, &client_history, ent);
231 const char *transport_name,
240 if (!dos_enabled()) {
252 log_debug(
LD_GENERAL,
"Seen client from '%s' with transport '%s'.",
254 transport_name ? transport_name :
"<no transport>");
256 ent = geoip_lookup_client(addr, transport_name, action);
258 ent = clientmap_entry_new(action, addr, transport_name);
259 HT_INSERT(clientmap, &client_history, ent);
261 if (now / 60 <= (
int)MAX_LAST_SEEN_IN_MINUTES && now >= 0)
273 increment_v3_ns_request((
country_t) country_idx);
282 time_t cutoff = *(time_t*)_cutoff / 60;
284 clientmap_entry_free(ent);
295 clientmap_HT_FOREACH_FN(&client_history,
296 remove_old_client_helper_,
304 geoip_lookup_client(
const tor_addr_t *addr,
const char *transport_name,
313 lookup.action = action;
314 lookup.transport_name = (
char *) transport_name;
316 return HT_FIND(clientmap, &client_history, &lookup);
322 oom_clean_client_entries(time_t cutoff)
327 for (ent = HT_START(clientmap, &client_history); ent; ent = ent_next) {
330 ent_next = HT_NEXT_RMV(clientmap, &client_history, ent);
331 bytes += clientmap_entry_size(entry);
332 clientmap_entry_free(entry);
334 ent_next = HT_NEXT(clientmap, &client_history, ent);
341 #define GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF (4 * 60 * 60)
343 #define GEOIP_CLIENT_CACHE_OOM_STEP (15 * 50)
350 geoip_client_cache_handle_oom(time_t now,
size_t min_remove_bytes)
353 size_t bytes_removed = 0;
360 k = WRITE_STATS_INTERVAL;
369 if (k <= GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF) {
374 bytes_removed += oom_clean_client_entries(cutoff);
375 k -= GEOIP_CLIENT_CACHE_OOM_STEP;
376 }
while (bytes_removed < min_remove_bytes);
378 return bytes_removed;
383 geoip_client_cache_total_allocation(
void)
385 return geoip_client_history_cache_size;
390 static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM];
397 static int arrays_initialized = 0;
400 if (!arrays_initialized) {
401 memset(ns_v3_responses, 0,
sizeof(ns_v3_responses));
402 arrays_initialized = 1;
405 ns_v3_responses[response]++;
411 #define MIN_IPS_TO_NOTE_COUNTRY 1
414 #define MIN_IPS_TO_NOTE_ANYTHING 1
417 #define IP_GRANULARITY 8
429 c_hist_compare_(
const void **_a,
const void **_b)
432 if (a->
total > b->total)
434 else if (a->
total < b->total)
437 return strcmp(a->
country, b->country);
443 #define DIRREQ_TIMEOUT (10*60)
457 unsigned int state:3;
459 unsigned int completed:1;
462 size_t response_size;
463 struct timeval completion_time;
476 return a->dirreq_id == b->dirreq_id && a->type == b->type;
483 unsigned u = (unsigned) entry->dirreq_id;
484 u += entry->type << 20;
507 old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry);
508 if (old_ent && old_ent != entry) {
509 log_warn(
LD_BUG,
"Error when putting directory request into local "
510 "map. There was already an entry for the same identifier.");
523 lookup.dirreq_id = dirreq_id;
524 return HT_FIND(dirreqmap, &dirreq_map, &lookup);
538 ent->dirreq_id = dirreq_id;
540 ent->response_size = response_size;
542 dirreq_map_put_(ent, type, dirreq_id);
558 ent = dirreq_map_get_(type, dirreq_id);
563 if (new_state - 1 != ent->state)
565 ent->state = new_state;
566 if ((type == DIRREQ_DIRECT &&
568 (type == DIRREQ_TUNNELED &&
581 unsigned granularity = IP_GRANULARITY;
583 strmap_t *transport_counts = strmap_new();
592 static const char* no_transport_str =
"<OR>";
596 char *the_string = NULL;
599 if (HT_EMPTY(&client_history))
613 log_debug(
LD_GENERAL,
"Starting iteration for transport history. %d clients.",
614 HT_SIZE(&client_history));
617 HT_FOREACH(ent, clientmap, &client_history) {
620 const char *transport_name = (*ent)->transport_name;
622 transport_name = no_transport_str;
625 ptr = strmap_get(transport_counts, transport_name);
626 val = (uintptr_t)ptr;
629 strmap_set(transport_counts, transport_name, ptr);
635 log_debug(
LD_GENERAL,
"Client from '%s' with transport '%s'. "
636 "I've now seen %d clients.",
637 safe_str_client(
fmt_addr(&(*ent)->addr)),
638 transport_name ? transport_name :
"<no transport>",
647 void *transport_count_ptr = strmap_get(transport_counts, transport_name);
648 uintptr_t transport_count = (uintptr_t) transport_count_ptr;
650 log_debug(
LD_GENERAL,
"We got %"PRIu64
" clients with transport '%s'.",
651 ((uint64_t)transport_count), transport_name);
656 (uint64_t)transport_count,
658 } SMARTLIST_FOREACH_END(transport_name);
662 log_debug(
LD_GENERAL,
"Final bridge-ip-transports string: '%s'", the_string);
665 strmap_free(transport_counts, NULL);
667 smartlist_free(transports_used);
669 smartlist_free(string_chunks);
685 uint32_t complete = 0,
timeouts = 0, running = 0;
691 for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) {
693 if (ent->type != type) {
694 next = HT_NEXT(dirreqmap, &dirreq_map, ptr);
697 if (ent->completed) {
700 next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
702 if (
tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT)
706 next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
711 #define DIR_REQ_GRANULARITY 4
713 DIR_REQ_GRANULARITY);
715 DIR_REQ_GRANULARITY);
717 DIR_REQ_GRANULARITY);
720 "running=%u", complete,
timeouts, running);
722 #define MIN_DIR_REQ_RESPONSES 16
723 if (complete >= MIN_DIR_REQ_RESPONSES) {
727 complete = smartlist_len(dirreq_completed);
728 dltimes = tor_calloc(complete,
sizeof(uint32_t));
730 uint32_t bytes_per_second;
731 uint32_t time_diff_ = (uint32_t)
tv_mdiff(&ent->request_time,
732 &ent->completion_time);
737 bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff_);
738 dltimes[ent_sl_idx] = bytes_per_second;
739 } SMARTLIST_FOREACH_END(ent);
740 median_uint32(dltimes, complete);
742 ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u,"
743 "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u",
745 dltimes[1*complete/10-1],
746 dltimes[2*complete/10-1],
747 dltimes[1*complete/4-1],
748 dltimes[3*complete/10-1],
749 dltimes[4*complete/10-1],
750 dltimes[5*complete/10-1],
751 dltimes[6*complete/10-1],
752 dltimes[7*complete/10-1],
753 dltimes[3*complete/4-1],
754 dltimes[8*complete/10-1],
755 dltimes[9*complete/10-1],
756 dltimes[complete-1]);
764 smartlist_free(dirreq_completed);
785 char **country_str,
char **ipver_str)
787 unsigned granularity = IP_GRANULARITY;
794 unsigned ipv4_count = 0, ipv6_count = 0;
799 counts = tor_calloc(n_countries,
sizeof(
unsigned));
800 HT_FOREACH(cm_ent, clientmap, &client_history) {
802 if ((*cm_ent)->action != (
int)action)
807 tor_assert(0 <= country && country < n_countries);
827 smartlist_free(chunks);
831 if (total < MIN_IPS_TO_NOTE_ANYTHING) {
840 for (i = 0; i < n_countries; ++i) {
842 const char *countrycode;
845 if (c >= MIN_IPS_TO_NOTE_COUNTRY) {
865 smartlist_free(chunks);
869 smartlist_free(entries);
883 unsigned granularity = IP_GRANULARITY;
889 if ((
size_t)c_sl_idx < n_v3_ns_requests_len)
890 tot = n_v3_ns_requests[c_sl_idx];
895 ent = tor_malloc_zero(
sizeof(
c_hist_t));
899 } SMARTLIST_FOREACH_END(c);
909 smartlist_free(strings);
910 smartlist_free(entries);
916 static time_t start_of_dirreq_stats_interval;
922 start_of_dirreq_stats_interval = now;
929 memset(n_v3_ns_requests, 0,
930 n_v3_ns_requests_len *
sizeof(uint32_t));
933 for (ent = HT_START(clientmap, &client_history); ent != NULL;
937 next = HT_NEXT_RMV(clientmap, &client_history, ent);
938 clientmap_entry_free(
this);
940 next = HT_NEXT(clientmap, &client_history, ent);
944 memset(ns_v3_responses, 0,
sizeof(ns_v3_responses));
947 for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
949 next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
953 start_of_dirreq_stats_interval = now;
970 char t[ISO_TIME_LEN+1];
972 char *v3_ips_string = NULL, *v3_reqs_string = NULL,
973 *v3_direct_dl_string = NULL, *v3_tunneled_dl_string = NULL;
976 if (!start_of_dirreq_stats_interval)
979 tor_assert(now >= start_of_dirreq_stats_interval);
985 #define RESPONSE_GRANULARITY 8
986 for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) {
988 ns_v3_responses[i], RESPONSE_GRANULARITY);
990 #undef RESPONSE_GRANULARITY
992 v3_direct_dl_string = geoip_get_dirreq_history(DIRREQ_DIRECT);
993 v3_tunneled_dl_string = geoip_get_dirreq_history(DIRREQ_TUNNELED);
998 "dirreq-v3-reqs %s\n"
999 "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u,"
1000 "not-found=%u,not-modified=%u,busy=%u\n"
1001 "dirreq-v3-direct-dl %s\n"
1002 "dirreq-v3-tunneled-dl %s\n",
1004 (
unsigned) (now - start_of_dirreq_stats_interval),
1005 v3_ips_string ? v3_ips_string :
"",
1006 v3_reqs_string ? v3_reqs_string :
"",
1013 v3_direct_dl_string ? v3_direct_dl_string :
"",
1014 v3_tunneled_dl_string ? v3_tunneled_dl_string :
"");
1035 if (!start_of_dirreq_stats_interval)
1037 if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now)
1057 return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL;
1062 static time_t start_of_bridge_stats_interval;
1068 start_of_bridge_stats_interval = now;
1076 client_history_clear();
1077 start_of_bridge_stats_interval = 0;
1084 validate_bridge_stats(
const char *stats_str, time_t now)
1086 char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1],
1089 const char *BRIDGE_STATS_END =
"bridge-stats-end ";
1090 const char *BRIDGE_IPS =
"bridge-ips ";
1091 const char *BRIDGE_IPS_EMPTY_LINE =
"bridge-ips\n";
1092 const char *BRIDGE_TRANSPORTS =
"bridge-ip-transports ";
1093 const char *BRIDGE_TRANSPORTS_EMPTY_LINE =
"bridge-ip-transports\n";
1095 time_t stats_end_time;
1104 tmp += strlen(BRIDGE_STATS_END);
1106 if (strlen(tmp) < ISO_TIME_LEN + 6)
1108 strlcpy(stats_end_str, tmp,
sizeof(stats_end_str));
1111 if (stats_end_time < now - (25*60*60) ||
1112 stats_end_time > now + (1*60*60))
1114 seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10);
1115 if (!eos || seconds < 23*60*60)
1142 static char *bridge_stats_extrainfo = NULL;
1151 char *country_data = NULL, *ipver_data = NULL, *transport_data = NULL;
1152 long duration = now - start_of_bridge_stats_interval;
1153 char written[ISO_TIME_LEN+1];
1157 if (!start_of_bridge_stats_interval)
1165 "bridge-stats-end %s (%ld s)\n"
1167 "bridge-ip-versions %s\n"
1168 "bridge-ip-transports %s\n",
1170 country_data ? country_data :
"",
1171 ipver_data ? ipver_data :
"",
1172 transport_data ? transport_data :
"");
1184 format_bridge_stats_controller(time_t now)
1186 char *out = NULL, *country_data = NULL, *ipver_data = NULL;
1187 char started[ISO_TIME_LEN+1];
1194 "TimeStarted=\"%s\" CountrySummary=%s IPVersions=%s",
1196 country_data ? country_data :
"",
1197 ipver_data ? ipver_data :
"");
1213 unsigned cutoff = (unsigned)( (now-n_seconds)/60 );
1215 if (!start_of_bridge_stats_interval)
1219 HT_FOREACH(ent, clientmap, &client_history) {
1223 if ((*ent)->last_seen_in_minutes < cutoff)
1229 "Since last heartbeat message, I have seen %d unique clients.",
1243 if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL)
1244 return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
1256 bridge_stats_extrainfo = val;
1257 start_of_bridge_stats_interval = now;
1262 bridge_stats_extrainfo,
"bridge statistics");
1266 char *controller_str = format_bridge_stats_controller(now);
1274 return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
1281 load_bridge_stats(time_t now)
1283 char *fname, *contents;
1284 if (bridge_stats_extrainfo)
1287 fname = get_datadir_fname2(
"stats",
"bridge-stats");
1289 if (contents && validate_bridge_stats(contents, now)) {
1290 bridge_stats_extrainfo = contents;
1303 load_bridge_stats(now);
1304 return bridge_stats_extrainfo;
1312 return format_bridge_stats_controller(now);
1317 static time_t start_of_entry_stats_interval;
1323 start_of_entry_stats_interval = now;
1330 client_history_clear();
1331 start_of_entry_stats_interval = now;
1348 char t[ISO_TIME_LEN+1];
1352 if (!start_of_entry_stats_interval)
1355 tor_assert(now >= start_of_entry_stats_interval);
1360 "entry-stats-end %s (%u s)\n"
1362 t, (
unsigned) (now - start_of_entry_stats_interval),
1377 if (!start_of_entry_stats_interval)
1379 if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now)
1398 return start_of_entry_stats_interval + WRITE_STATS_INTERVAL;
1407 for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) {
1409 next = HT_NEXT_RMV(clientmap, &client_history, ent);
1410 clientmap_entry_free(
this);
1412 HT_CLEAR(clientmap, &client_history);
1416 for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
1418 next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
1421 HT_CLEAR(dirreqmap, &dirreq_map);
uint64_t tor_addr_hash(const tor_addr_t *addr)
void tor_addr_copy(tor_addr_t *dest, const tor_addr_t *src)
int tor_addr_compare(const tor_addr_t *addr1, const tor_addr_t *addr2, tor_addr_comparison_t how)
static sa_family_t tor_addr_family(const tor_addr_t *a)
void buf_add_printf(buf_t *buf, const char *format,...)
buf_t * buf_new_with_capacity(size_t size)
char * buf_extract(buf_t *buf, size_t *sz_out)
Header file for buffers.c.
const or_options_t * get_options(void)
int check_or_create_data_subdir(const char *subdir)
int write_to_data_subdir(const char *subdir, const char *fname, const char *str, const char *descr)
Header file for config.c.
static conn_counts_t counts
void control_event_clients_seen(const char *controller_str)
Header file for control_events.c.
Header file for dnsserv.c.
#define RFTS_IGNORE_MISSING
int geoip_is_loaded(sa_family_t family)
const char * geoip_get_country_name(country_t num)
int geoip_get_country_by_addr(const tor_addr_t *addr)
const smartlist_t * geoip_get_countries(void)
int geoip_get_n_countries(void)
Header file for geoip_stats.c.
time_t geoip_entry_stats_write(time_t now)
int geoip_get_client_history(geoip_client_action_t action, char **country_str, char **ipver_str)
void geoip_reset_entry_stats(time_t now)
char * geoip_format_bridge_stats(time_t now)
time_t geoip_dirreq_stats_write(time_t now)
@ DIRREQ_CHANNEL_BUFFER_FLUSHED
@ DIRREQ_IS_FOR_NETWORK_STATUS
@ DIRREQ_FLUSHING_DIR_CONN_FINISHED
void geoip_dirreq_stats_init(time_t now)
void geoip_bridge_stats_init(time_t now)
char * geoip_get_request_history(void)
void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, dirreq_state_t new_state)
void geoip_note_ns_response(geoip_ns_response_t response)
void geoip_remove_old_clients(time_t cutoff)
void geoip_bridge_stats_term(void)
void geoip_entry_stats_init(time_t now)
void geoip_dirreq_stats_term(void)
@ GEOIP_CLIENT_NETWORKSTATUS
char * geoip_format_dirreq_stats(time_t now)
char * geoip_get_bridge_stats_controller(time_t)
void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, dirreq_type_t type)
void geoip_note_client_seen(geoip_client_action_t action, const tor_addr_t *addr, const char *transport_name, time_t now)
char * format_client_stats_heartbeat(time_t now)
char * geoip_get_transport_history(void)
void geoip_reset_dirreq_stats(time_t now)
char * geoip_format_entry_stats(time_t now)
const char * geoip_get_bridge_stats_extrainfo(time_t)
void geoip_stats_free_all(void)
void geoip_entry_stats_term(void)
int should_record_bridge_info(const or_options_t *options)
time_t geoip_bridge_stats_write(time_t now)
@ GEOIP_REJECT_NOT_MODIFIED
@ GEOIP_REJECT_NOT_ENOUGH_SIGS
@ GEOIP_REJECT_UNAVAILABLE
HT_PROTOTYPE(hs_circuitmap_ht, circuit_t, hs_circuitmap_node, hs_circuit_hash_token, hs_circuits_have_same_token)
typedef HT_HEAD(hs_service_ht, hs_service_t) hs_service_ht
void tor_free_(void *mem)
void * tor_reallocarray_(void *ptr, size_t sz1, size_t sz2)
uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor)
unsigned round_to_next_multiple_of(unsigned number, unsigned divisor)
uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor)
Master header file for Tor-specific functionality.
int tor_asprintf(char **strp, const char *fmt,...)
Header file for routerlist.c.
void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern,...)
void smartlist_sort_strings(smartlist_t *sl)
char * smartlist_join_strings(smartlist_t *sl, const char *join, int terminate, size_t *len_out)
void smartlist_sort(smartlist_t *sl, int(*compare)(const void **a, const void **b))
smartlist_t * smartlist_new(void)
void smartlist_add_strdup(struct smartlist_t *sl, const char *string)
void smartlist_add(smartlist_t *sl, void *element)
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
#define SMARTLIST_FOREACH(sl, type, var, cmd)
unsigned int last_seen_in_minutes
int BridgeRecordUsageByCountry
int BridgeAuthoritativeDir
int parse_iso_time(const char *cp, time_t *t)
void format_iso_time(char *buf, time_t t)
void tor_gettimeofday(struct timeval *timeval)
long tv_mdiff(const struct timeval *start, const struct timeval *end)
#define IF_BUG_ONCE(cond)
int strcmp_opt(const char *s1, const char *s2)
const char * find_str_at_start_of_line(const char *haystack, const char *needle)