40 #ifdef HAVE_SYS_STAT_H
65 uint64_t total_len_seen;
74 static void warn_if_nul_found(
const char *inp,
size_t len, int64_t offset,
75 const char *activity);
78 static inline unsigned int
81 return (
unsigned) siphash24g(md->
digest,
sizeof(md->
digest));
102 #define TOO_MANY_OUTDATED_DIRSERVERS 30
133 log_info(
LD_GENERAL,
"Too many outdated directory servers (%d). Resetting.",
145 log_info(
LD_GENERAL,
"Auth %s gave us outdated dirinfo.", relay_hexdigest);
157 log_info(
LD_GENERAL,
"Noted %s as outdated md dirserver", relay_hexdigest);
207 if (md->
body == NULL) {
208 *annotation_len_out = 0;
213 char buf[ISO_TIME_LEN+1];
214 char annotation[ISO_TIME_LEN+32];
216 tor_snprintf(annotation,
sizeof(annotation),
"@last-listed %s\n", buf);
219 "Couldn't write microdescriptor annotation: %s",
223 r += strlen(annotation);
224 *annotation_len_out = r;
226 *annotation_len_out = 0;
231 "dumping a microdescriptor");
233 if (written != (ssize_t)md->
bodylen) {
234 written = written < 0 ? 0 : written;
236 "Couldn't dump microdescriptor (wrote %ld out of %lu): %s",
237 (
long)written, (
unsigned long)md->
bodylen,
254 if (PREDICT_UNLIKELY(cache->is_loaded == 0)) {
267 HT_INIT(microdesc_map, &cache->map);
268 cache->cache_fname = get_cachedir_fname(
"cached-microdescs");
269 cache->journal_fname = get_cachedir_fname(
"cached-microdescs.new");
295 int no_save, time_t listed_at,
298 void *
const DIGEST_REQUESTED = (
void*)1;
299 void *
const DIGEST_RECEIVED = (
void*)2;
300 void *
const DIGEST_INVALID = (
void*)3;
308 where, invalid_digests);
309 if (listed_at != (time_t)-1) {
313 if (requested_digests256) {
314 digest256map_t *requested;
315 requested = digest256map_new();
318 digest256map_set(requested, cp, DIGEST_REQUESTED));
324 if (digest256map_get(requested, cp)) {
325 digest256map_set(requested, cp, DIGEST_INVALID);
330 } SMARTLIST_FOREACH_END(cp);
335 if (digest256map_get(requested, (
const uint8_t*)md->digest)) {
336 digest256map_set(requested, (
const uint8_t*)md->digest,
339 log_fn(LOG_PROTOCOL_WARN,
LD_DIR,
"Received non-requested microdesc");
343 } SMARTLIST_FOREACH_END(md);
347 void *status = digest256map_get(requested, cp);
348 if (status == DIGEST_RECEIVED || status == DIGEST_INVALID) {
352 } SMARTLIST_FOREACH_END(cp);
353 digest256map_free(requested, NULL);
358 if (smartlist_len(invalid_digests)) {
368 } SMARTLIST_FOREACH_END(d);
372 smartlist_free(invalid_digests);
375 smartlist_free(descriptors);
395 OPEN_FLAGS_APPEND|O_BINARY,
398 log_warn(
LD_DIR,
"Couldn't append to journal in %s: %s",
399 cache->journal_fname, strerror(errno));
406 md2 = HT_FIND(microdesc_map, &cache->map, md);
413 cache->bytes_dropped += size;
419 size_t annotation_len;
427 cache->journal_len += size;
430 md->saved_location = where;
433 md->no_save = no_save;
435 HT_INSERT(microdesc_map, &cache->map, md);
439 cache->total_len_seen += md->bodylen;
440 } SMARTLIST_FOREACH_END(md);
444 log_warn(
LD_DIR,
"Error appending to microdescriptor file: %s",
453 if (ns && ns->
flavor == FLAV_MICRODESC)
457 if (smartlist_len(added))
469 for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
471 next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
475 HT_CLEAR(microdesc_map, &cache->map);
476 if (cache->cache_content) {
477 int res = tor_munmap_file(cache->cache_content);
480 "tor_munmap_file() failed clearing microdesc cache; "
481 "we are probably about to leak memory.");
484 cache->cache_content = NULL;
486 cache->total_len_seen = 0;
488 cache->bytes_dropped = 0;
492 warn_if_nul_found(
const char *inp,
size_t len, int64_t offset,
493 const char *activity)
495 const char *nul_found = memchr(inp, 0, len);
496 if (BUG(nul_found)) {
497 log_warn(
LD_BUG,
"Found unexpected NUL while %s, offset %"PRId64
498 "at position %"TOR_PRIuSZ
"/%"TOR_PRIuSZ
".",
499 activity, offset, (nul_found - inp), len);
500 const char *start_excerpt_at, *eos = inp + len;
501 if ((nul_found - inp) >= 16)
502 start_excerpt_at = nul_found - 16;
504 start_excerpt_at = inp;
505 size_t excerpt_len = MIN(32, eos - start_excerpt_at);
507 base16_encode(tmp,
sizeof(tmp), start_excerpt_at, excerpt_len);
508 log_warn(
LD_BUG,
" surrounding string: %s", tmp);
518 char *journal_content;
525 cache->is_loaded = 1;
527 mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
529 warn_if_nul_found(mm->
data, mm->
size, 0,
"scanning microdesc cache");
533 total += smartlist_len(added);
534 smartlist_free(added);
538 journal_content = read_file_to_str(cache->journal_fname,
540 if (journal_content) {
541 cache->journal_len = strlen(journal_content);
542 warn_if_nul_found(journal_content, (
size_t)st.st_size, 0,
543 "reading microdesc journal");
545 journal_content+st.st_size,
548 total += smartlist_len(added);
549 smartlist_free(added);
553 log_info(
LD_DIR,
"Reloaded microdescriptor cache. Found %d descriptors.",
563 #define TOLERATE_MICRODESC_AGE (7*24*60*60)
575 int dropped=0, kept=0;
576 size_t bytes_dropped = 0;
577 time_t now = time(NULL);
588 for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
589 const int is_old = (*mdp)->last_listed < cutoff;
590 const unsigned held_by_nodes = (*mdp)->held_by_nodes;
591 if (is_old && !held_by_nodes) {
594 mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
596 bytes_dropped += victim->
bodylen;
597 microdesc_free(victim);
604 long networkstatus_age = -1;
605 const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
609 log_warn(
LD_BUG,
"Microdescriptor seemed very old "
610 "(last listed %d hours ago vs %d hour cutoff), but is still "
611 "marked as being held by %d node(s). I found %d node(s) "
612 "holding it. Current networkstatus is %ld hours old. "
613 "Hashtable badness is %d.",
614 (
int)((now - (*mdp)->last_listed) / 3600),
615 (
int)((now - cutoff) / 3600),
617 smartlist_len(nodes),
618 networkstatus_age / 3600,
622 const char *rs_match =
"No RS";
623 const char *rs_present =
"";
625 if (
tor_memeq(node->rs->descriptor_digest,
627 rs_match =
"Microdesc digest in RS matches";
629 rs_match =
"Microdesc digest in RS does match";
633 rs_present =
" RS not present in networkstatus.";
635 if (rs == node->rs) {
636 rs_present =
" RS okay in networkstatus.";
641 log_warn(
LD_BUG,
" [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s",
644 node->md, node->rs, node->ri, rs_match, rs_present);
645 } SMARTLIST_FOREACH_END(node);
646 smartlist_free(nodes);
647 (*mdp)->last_listed = now;
651 mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
656 log_info(
LD_DIR,
"Removed %d/%d microdescriptors as old.",
657 dropped,dropped+kept);
658 cache->bytes_dropped += bytes_dropped;
665 const size_t old_len =
666 cache->cache_content ? cache->cache_content->size : 0;
667 const size_t journal_len = cache->journal_len;
668 const size_t dropped = cache->bytes_dropped;
670 if (journal_len < 16384)
672 if (dropped > (journal_len + old_len) / 3)
674 if (journal_len > old_len / 2)
712 off_t off = 0, off_real;
713 int orig_size, new_size;
724 if (!force && !should_rebuild_md_cache(cache))
727 log_info(
LD_DIR,
"Rebuilding the microdescriptor cache...");
729 orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
730 orig_size += (int)cache->journal_len;
733 OPEN_FLAGS_REPLACE|O_BINARY,
740 HT_FOREACH(mdp, microdesc_map, &cache->map) {
742 size_t annotation_len;
755 md->
off = off + annotation_len;
758 if (off_real != off) {
759 log_warn(
LD_BUG,
"Discontinuity in position in microdescriptor cache."
760 "By my count, I'm at %"PRId64
761 ", but I should be at %"PRId64,
762 (int64_t)(off), (int64_t)(off_real));
775 if (cache->cache_content) {
776 res = tor_munmap_file(cache->cache_content);
779 "Failed to unmap old microdescriptor cache while rebuilding");
781 cache->cache_content = NULL;
785 log_warn(
LD_DIR,
"Error rebuilding microdescriptor cache: %s",
788 cache->cache_content = NULL;
789 HT_FOREACH(mdp, microdesc_map, &cache->map) {
795 smartlist_free(wrote);
799 cache->cache_content = tor_mmap_file(cache->cache_fname);
801 if (!cache->cache_content && smartlist_len(wrote)) {
802 log_err(
LD_DIR,
"Couldn't map file that we just wrote to %s!",
804 smartlist_free(wrote);
809 md->body = (
char*)cache->cache_content->data + md->off;
810 if (PREDICT_UNLIKELY(
811 md->bodylen < 9 ||
fast_memneq(md->body,
"onion-key", 9) != 0)) {
814 off_t avail = cache->cache_content->size - md->off;
817 bad_str = tor_strndup(md->body, MIN(128, (
size_t)avail));
818 log_err(
LD_BUG,
"After rebuilding microdesc cache, offsets seem wrong. "
819 " At offset %d, I expected to find a microdescriptor starting "
820 " with \"onion-key\". Instead I got %s.",
821 (
int)md->off,
escaped(bad_str));
825 } SMARTLIST_FOREACH_END(md);
827 smartlist_free(wrote);
830 cache->journal_len = 0;
831 cache->bytes_dropped = 0;
833 new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
834 log_info(
LD_DIR,
"Done rebuilding microdesc cache. "
835 "Saved %d bytes; %d still used.",
836 orig_size-new_size, new_size);
852 unsigned int found=0;
855 if (node->md == md) {
875 microdesc_t *md2 = HT_FIND(microdesc_map, &cache->map, md);
877 log_warn(
LD_BUG,
"microdesc_free() called from %s:%d, but md was still "
878 "in microdesc_map", fname, lineno);
879 HT_REMOVE(microdesc_map, &cache->map, md);
881 log_warn(
LD_BUG,
"microdesc_free() called from %s:%d with held_in_map "
882 "set, but microdesc was not in the map.", fname, lineno);
890 const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
892 if (node->md == md) {
898 log_warn(
LD_BUG,
"microdesc_free() called from %s:%d, but md was still "
899 "referenced %d node(s); held_by_nodes == %u, ht_badness == %d",
902 log_warn(
LD_BUG,
"microdesc_free() called from %s:%d with held_by_nodes "
903 "set to %u, but md was not referenced by any nodes. "
919 nodefamily_free(md->
family);
952 md = HT_FIND(microdesc_map, &cache->map, &search);
962 int downloadable_only, digest256map_t *skip)
965 time_t now = time(NULL);
970 if (downloadable_only &&
973 if (skip && digest256map_get(skip, (
const uint8_t*)rs->descriptor_digest))
981 } SMARTLIST_FOREACH_END(rs);
998 digest256map_t *pending;
1013 pending = digest256map_new();
1020 digest256map_free(pending, NULL);
1023 missing, NULL, now);
1025 smartlist_free(missing);
1049 } SMARTLIST_FOREACH_END(rs);
1089 return FLAV_MICRODESC;
const char * hex_str(const char *from, size_t fromlen)
void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
Header file for circuitbuild.c.
const or_options_t * get_options(void)
Header file for config.c.
int tor_memeq(const void *a, const void *b, size_t sz)
#define fast_memeq(a, b, c)
#define fast_memneq(a, b, c)
int dirclient_too_idle_to_fetch_descriptors(const or_options_t *options, time_t now)
Header for feature/dirclient/dirclient_modes.c.
Header file for directory.c.
#define DIR_PURPOSE_FETCH_MICRODESC
dir_server_t * router_get_trusteddirserver_by_digest(const char *digest)
Header file for dirlist.c.
int directory_caches_dir_info(const or_options_t *options)
Header file for dirserv.c.
int download_status_is_ready(download_status_t *dls, time_t now)
void download_status_mark_impossible(download_status_t *dl)
Header file for dlstatus.c.
Header file for circuitbuild.c.
const char * escaped(const char *s)
off_t tor_fd_getpos(int fd)
int tor_fd_setpos(int fd, off_t pos)
int write_str_to_file(const char *fname, const char *str, int bin)
#define RFTS_IGNORE_MISSING
int finish_writing_to_file(open_file_t *file_data)
int start_writing_to_file(const char *fname, int open_flags, int mode, open_file_t **data_out)
ssize_t write_all_to_fd(int fd, const char *buf, size_t count)
int abort_writing_to_file(open_file_t *file_data)
HT_PROTOTYPE(hs_circuitmap_ht, circuit_t, hs_circuitmap_node, hs_circuit_hash_token, hs_circuits_have_same_token)
typedef HT_HEAD(hs_service_ht, hs_service_t) hs_service_ht
#define log_fn(severity, domain, args,...)
void tor_free_(void *mem)
void * tor_reallocarray_(void *ptr, size_t sz1, size_t sz2)
static int microdesc_eq_(microdesc_t *a, microdesc_t *b)
void microdesc_free_(microdesc_t *md, const char *fname, int lineno)
void microdesc_reset_outdated_dirservers_list(void)
void microdesc_free_all(void)
int usable_consensus_flavor(void)
static unsigned int microdesc_hash_(microdesc_t *md)
int we_fetch_router_descriptors(const or_options_t *options)
void microdesc_note_outdated_dirserver(const char *relay_digest)
smartlist_t * microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, int no_save)
void microdesc_check_counts(void)
static microdesc_cache_t * get_microdesc_cache_noload(void)
int microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
static smartlist_t * outdated_dirserver_list
void microdesc_cache_clear(microdesc_cache_t *cache)
smartlist_t * microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, int downloadable_only, digest256map_t *skip)
static ssize_t dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
int microdesc_cache_reload(microdesc_cache_t *cache)
microdesc_t * microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
smartlist_t * microdescs_add_to_cache(microdesc_cache_t *cache, const char *s, const char *eos, saved_location_t where, int no_save, time_t listed_at, smartlist_t *requested_digests256)
microdesc_cache_t * get_microdesc_cache(void)
static void microdesc_wipe_body(microdesc_t *md)
#define TOLERATE_MICRODESC_AGE
void update_microdesc_downloads(time_t now)
int microdesc_relay_is_outdated_dirserver(const char *relay_digest)
void update_microdescs_from_networkstatus(time_t now)
static microdesc_cache_t * the_microdesc_cache
int we_fetch_microdescriptors(const or_options_t *options)
int we_use_microdescriptors_for_circuits(const or_options_t *options)
void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
Header file for microdesc.c.
smartlist_t * microdescs_parse_from_string(const char *s, const char *eos, int allow_annotations, saved_location_t where, smartlist_t *invalid_digests_out)
Header file for microdesc_parse.c.
Microdescriptor structure.
networkstatus_t * networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
networkstatus_t * networkstatus_get_reasonably_live_consensus(time_t now, int flavor)
networkstatus_t * networkstatus_get_latest_consensus(void)
int should_delay_dir_fetches(const or_options_t *options, const char **msg_out)
routerstatus_t * router_get_mutable_consensus_status_by_descriptor_digest(networkstatus_t *consensus, const char *digest)
Header file for networkstatus.c.
Networkstatus consensus/vote structure.
Node information structure.
Header file for nodefamily.c.
void router_dir_info_changed(void)
const smartlist_t * nodelist_get_list(void)
smartlist_t * nodelist_find_nodes_with_microdesc(const microdesc_t *md)
node_t * nodelist_add_microdesc(microdesc_t *md)
Header file for nodelist.c.
Master header file for Tor-specific functionality.
Header file for policies.c.
int tor_snprintf(char *str, size_t size, const char *format,...)
Header file for router.c.
void list_pending_microdesc_downloads(digest256map_t *result)
void launch_descriptor_downloads(int purpose, smartlist_t *downloadable, const routerstatus_t *source, time_t now)
Header file for routerlist.c.
Routerstatus (consensus entry) structure.
int smartlist_contains_string(const smartlist_t *sl, const char *element)
smartlist_t * smartlist_new(void)
void smartlist_add_strdup(struct smartlist_t *sl, const char *string)
void smartlist_add(smartlist_t *sl, void *element)
void smartlist_clear(smartlist_t *sl)
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
#define SMARTLIST_FOREACH(sl, type, var, cmd)
#define SMARTLIST_DEL_CURRENT(sl, var)
unsigned int held_by_nodes
struct short_policy_t * exit_policy
saved_location_bitfield_t saved_location
struct curve25519_public_key_t * onion_curve25519_pkey
char digest[DIGEST256_LEN]
struct nodefamily_t * family
struct ed25519_public_key_t * ed25519_identity_pkey
struct short_policy_t * ipv6_exit_policy
smartlist_t * routerstatus_list
consensus_flavor_t flavor
int FetchUselessDescriptors
char descriptor_digest[DIGEST256_LEN]
#define MOCK_IMPL(rv, funcname, arglist)
void format_iso_time(char *buf, time_t t)
#define tor_fragile_assert()
int fast_mem_is_zero(const char *mem, size_t len)