tor  0.4.2.0-alpha-dev
microdesc.c
Go to the documentation of this file.
1 /* Copyright (c) 2009-2019, The Tor Project, Inc. */
2 /* See LICENSE for licensing information */
3 
11 #include "core/or/or.h"
12 
13 #include "lib/fdio/fdio.h"
14 
15 #include "app/config/config.h"
16 #include "core/or/circuitbuild.h"
17 #include "core/or/policies.h"
29 #include "feature/relay/router.h"
30 
31 #include "feature/nodelist/microdesc_st.h"
32 #include "feature/nodelist/networkstatus_st.h"
33 #include "feature/nodelist/node_st.h"
34 #include "feature/nodelist/routerstatus_st.h"
35 
36 #ifdef HAVE_FCNTL_H
37 #include <fcntl.h>
38 #endif
39 #ifdef HAVE_SYS_STAT_H
40 #include <sys/stat.h>
41 #endif
42 
50  HT_HEAD(microdesc_map, microdesc_t) map;
51 
53  char *cache_fname;
55  char *journal_fname;
57  tor_mmap_t *cache_content;
59  size_t journal_len;
61  size_t bytes_dropped;
62 
64  uint64_t total_len_seen;
66  unsigned n_seen;
67 
69  int is_loaded;
70 };
71 
73 static void warn_if_nul_found(const char *inp, size_t len, int64_t offset,
74  const char *activity);
75 
77 static inline unsigned int
79 {
80  return (unsigned) siphash24g(md->digest, sizeof(md->digest));
81 }
82 
85 static inline int
87 {
88  return tor_memeq(a->digest, b->digest, DIGEST256_LEN);
89 }
90 
91 HT_PROTOTYPE(microdesc_map, microdesc_t, node,
93 HT_GENERATE2(microdesc_map, microdesc_t, node,
96 
97 /************************* md fetch fail cache *****************************/
98 
99 /* If we end up with too many outdated dirservers, something probably went
100  * wrong so clean up the list. */
101 #define TOO_MANY_OUTDATED_DIRSERVERS 30
102 
105 static smartlist_t *outdated_dirserver_list = NULL;
106 
109 void
110 microdesc_note_outdated_dirserver(const char *relay_digest)
111 {
112  char relay_hexdigest[HEX_DIGEST_LEN+1];
113 
114  /* If we have a reasonably live consensus, then most of our dirservers should
115  * still be caching all the microdescriptors in it. Reasonably live
116  * consensuses are up to a day old (or a day in the future). But
117  * microdescriptors expire 7 days after the last consensus that referenced
118  * them. */
119  if (!networkstatus_get_reasonably_live_consensus(approx_time(),
120  FLAV_MICRODESC)) {
121  return;
122  }
123 
124  if (!outdated_dirserver_list) {
125  outdated_dirserver_list = smartlist_new();
126  }
127 
128  tor_assert(outdated_dirserver_list);
129 
130  /* If the list grows too big, clean it up */
131  if (BUG(smartlist_len(outdated_dirserver_list) >
132  TOO_MANY_OUTDATED_DIRSERVERS)) {
134  }
135 
136  /* Turn the binary relay digest to a hex since smartlists have better support
137  * for strings than digests. */
138  base16_encode(relay_hexdigest,sizeof(relay_hexdigest),
139  relay_digest, DIGEST_LEN);
140 
141  /* Make sure we don't add a dirauth as an outdated dirserver */
142  if (router_get_trusteddirserver_by_digest(relay_digest)) {
143  log_info(LD_GENERAL, "Auth %s gave us outdated dirinfo.", relay_hexdigest);
144  return;
145  }
146 
147  /* Don't double-add outdated dirservers */
148  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
149  return;
150  }
151 
152  /* Add it to the list of outdated dirservers */
153  smartlist_add_strdup(outdated_dirserver_list, relay_hexdigest);
154 
155  log_info(LD_GENERAL, "Noted %s as outdated md dirserver", relay_hexdigest);
156 }
157 
160 int
161 microdesc_relay_is_outdated_dirserver(const char *relay_digest)
162 {
163  char relay_hexdigest[HEX_DIGEST_LEN+1];
164 
165  if (!outdated_dirserver_list) {
166  return 0;
167  }
168 
169  /* Convert identity digest to hex digest */
170  base16_encode(relay_hexdigest, sizeof(relay_hexdigest),
171  relay_digest, DIGEST_LEN);
172 
173  /* Last time we tried to fetch microdescs, was this directory mirror missing
174  * any mds we asked for? */
175  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
176  return 1;
177  }
178 
179  return 0;
180 }
181 
183 void
185 {
186  if (!outdated_dirserver_list) {
187  return;
188  }
189 
190  SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
191  smartlist_clear(outdated_dirserver_list);
192 }
193 
194 /****************************************************************************/
195 
200 static ssize_t
201 dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
202 {
203  ssize_t r = 0;
204  ssize_t written;
205  if (md->body == NULL) {
206  *annotation_len_out = 0;
207  return 0;
208  }
209  /* XXXX drops unknown annotations. */
210  if (md->last_listed) {
211  char buf[ISO_TIME_LEN+1];
212  char annotation[ISO_TIME_LEN+32];
213  format_iso_time(buf, md->last_listed);
214  tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
215  if (write_all_to_fd(fd, annotation, strlen(annotation)) < 0) {
216  log_warn(LD_DIR,
217  "Couldn't write microdescriptor annotation: %s",
218  strerror(errno));
219  return -1;
220  }
221  r += strlen(annotation);
222  *annotation_len_out = r;
223  } else {
224  *annotation_len_out = 0;
225  }
226 
227  md->off = tor_fd_getpos(fd);
228  warn_if_nul_found(md->body, md->bodylen, (int64_t) md->off,
229  "dumping a microdescriptor");
230  written = write_all_to_fd(fd, md->body, md->bodylen);
231  if (written != (ssize_t)md->bodylen) {
232  written = written < 0 ? 0 : written;
233  log_warn(LD_DIR,
234  "Couldn't dump microdescriptor (wrote %ld out of %lu): %s",
235  (long)written, (unsigned long)md->bodylen,
236  strerror(errno));
237  return -1;
238  }
239  r += md->bodylen;
240  return r;
241 }
242 
246 
250 {
252  if (PREDICT_UNLIKELY(cache->is_loaded == 0)) {
253  microdesc_cache_reload(cache);
254  }
255  return cache;
256 }
257 
260 static microdesc_cache_t *
262 {
263  if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
264  microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache));
265  HT_INIT(microdesc_map, &cache->map);
266  cache->cache_fname = get_cachedir_fname("cached-microdescs");
267  cache->journal_fname = get_cachedir_fname("cached-microdescs.new");
268  the_microdesc_cache = cache;
269  }
270  return the_microdesc_cache;
271 }
272 
273 /* There are three sources of microdescriptors:
274  1) Generated by us while acting as a directory authority.
275  2) Loaded from the cache on disk.
276  3) Downloaded.
277 */
278 
290 smartlist_t *
292  const char *s, const char *eos, saved_location_t where,
293  int no_save, time_t listed_at,
294  smartlist_t *requested_digests256)
295 {
296  void * const DIGEST_REQUESTED = (void*)1;
297  void * const DIGEST_RECEIVED = (void*)2;
298  void * const DIGEST_INVALID = (void*)3;
299 
300  smartlist_t *descriptors, *added;
301  const int allow_annotations = (where != SAVED_NOWHERE);
302  smartlist_t *invalid_digests = smartlist_new();
303 
304  descriptors = microdescs_parse_from_string(s, eos,
305  allow_annotations,
306  where, invalid_digests);
307  if (listed_at != (time_t)-1) {
308  SMARTLIST_FOREACH(descriptors, microdesc_t *, md,
309  md->last_listed = listed_at);
310  }
311  if (requested_digests256) {
312  digest256map_t *requested;
313  requested = digest256map_new();
314  /* Set requested[d] to DIGEST_REQUESTED for every md we requested. */
315  SMARTLIST_FOREACH(requested_digests256, const uint8_t *, cp,
316  digest256map_set(requested, cp, DIGEST_REQUESTED));
317  /* Set requested[d] to DIGEST_INVALID for every md we requested which we
318  * will never be able to parse. Remove the ones we didn't request from
319  * invalid_digests.
320  */
321  SMARTLIST_FOREACH_BEGIN(invalid_digests, uint8_t *, cp) {
322  if (digest256map_get(requested, cp)) {
323  digest256map_set(requested, cp, DIGEST_INVALID);
324  } else {
325  tor_free(cp);
326  SMARTLIST_DEL_CURRENT(invalid_digests, cp);
327  }
328  } SMARTLIST_FOREACH_END(cp);
329  /* Update requested[d] to 2 for the mds we asked for and got. Delete the
330  * ones we never requested from the 'descriptors' smartlist.
331  */
332  SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
333  if (digest256map_get(requested, (const uint8_t*)md->digest)) {
334  digest256map_set(requested, (const uint8_t*)md->digest,
335  DIGEST_RECEIVED);
336  } else {
337  log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microdesc");
338  microdesc_free(md);
339  SMARTLIST_DEL_CURRENT(descriptors, md);
340  }
341  } SMARTLIST_FOREACH_END(md);
342  /* Remove the ones we got or the invalid ones from requested_digests256.
343  */
344  SMARTLIST_FOREACH_BEGIN(requested_digests256, uint8_t *, cp) {
345  void *status = digest256map_get(requested, cp);
346  if (status == DIGEST_RECEIVED || status == DIGEST_INVALID) {
347  tor_free(cp);
348  SMARTLIST_DEL_CURRENT(requested_digests256, cp);
349  }
350  } SMARTLIST_FOREACH_END(cp);
351  digest256map_free(requested, NULL);
352  }
353 
354  /* For every requested microdescriptor that was unparseable, mark it
355  * as not to be retried. */
356  if (smartlist_len(invalid_digests)) {
357  networkstatus_t *ns =
358  networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
359  if (ns) {
360  SMARTLIST_FOREACH_BEGIN(invalid_digests, char *, d) {
361  routerstatus_t *rs =
362  router_get_mutable_consensus_status_by_descriptor_digest(ns, d);
363  if (rs && tor_memeq(d, rs->descriptor_digest, DIGEST256_LEN)) {
364  download_status_mark_impossible(&rs->dl_status);
365  }
366  } SMARTLIST_FOREACH_END(d);
367  }
368  }
369  SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
370  smartlist_free(invalid_digests);
371 
372  added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
373  smartlist_free(descriptors);
374  return added;
375 }
376 
380 smartlist_t *
382  smartlist_t *descriptors, saved_location_t where,
383  int no_save)
384 {
385  smartlist_t *added;
386  open_file_t *open_file = NULL;
387  int fd = -1;
388  // int n_added = 0;
389  ssize_t size = 0;
390 
391  if (where == SAVED_NOWHERE && !no_save) {
392  fd = start_writing_to_file(cache->journal_fname,
393  OPEN_FLAGS_APPEND|O_BINARY,
394  0600, &open_file);
395  if (fd < 0) {
396  log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
397  cache->journal_fname, strerror(errno));
398  }
399  }
400 
401  added = smartlist_new();
402  SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
403  microdesc_t *md2;
404  md2 = HT_FIND(microdesc_map, &cache->map, md);
405  if (md2) {
406  /* We already had this one. */
407  if (md2->last_listed < md->last_listed)
408  md2->last_listed = md->last_listed;
409  microdesc_free(md);
410  if (where != SAVED_NOWHERE)
411  cache->bytes_dropped += size;
412  continue;
413  }
414 
415  /* Okay, it's a new one. */
416  if (fd >= 0) {
417  size_t annotation_len;
418  size = dump_microdescriptor(fd, md, &annotation_len);
419  if (size < 0) {
420  /* we already warned in dump_microdescriptor */
421  abort_writing_to_file(open_file);
422  fd = -1;
423  } else {
424  md->saved_location = SAVED_IN_JOURNAL;
425  cache->journal_len += size;
426  }
427  } else {
428  md->saved_location = where;
429  }
430 
431  md->no_save = no_save;
432 
433  HT_INSERT(microdesc_map, &cache->map, md);
434  md->held_in_map = 1;
435  smartlist_add(added, md);
436  ++cache->n_seen;
437  cache->total_len_seen += md->bodylen;
438  } SMARTLIST_FOREACH_END(md);
439 
440  if (fd >= 0) {
441  if (finish_writing_to_file(open_file) < 0) {
442  log_warn(LD_DIR, "Error appending to microdescriptor file: %s",
443  strerror(errno));
444  smartlist_clear(added);
445  return added;
446  }
447  }
448 
449  {
450  networkstatus_t *ns = networkstatus_get_latest_consensus();
451  if (ns && ns->flavor == FLAV_MICRODESC)
453  }
454 
455  if (smartlist_len(added))
457 
458  return added;
459 }
460 
462 void
464 {
465  microdesc_t **entry, **next;
466 
467  for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
468  microdesc_t *md = *entry;
469  next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
470  md->held_in_map = 0;
471  microdesc_free(md);
472  }
473  HT_CLEAR(microdesc_map, &cache->map);
474  if (cache->cache_content) {
475  int res = tor_munmap_file(cache->cache_content);
476  if (res != 0) {
477  log_warn(LD_FS,
478  "tor_munmap_file() failed clearing microdesc cache; "
479  "we are probably about to leak memory.");
480  /* TODO something smarter? */
481  }
482  cache->cache_content = NULL;
483  }
484  cache->total_len_seen = 0;
485  cache->n_seen = 0;
486  cache->bytes_dropped = 0;
487 }
488 
489 static void
490 warn_if_nul_found(const char *inp, size_t len, int64_t offset,
491  const char *activity)
492 {
493  const char *nul_found = memchr(inp, 0, len);
494  if (BUG(nul_found)) {
495  log_warn(LD_BUG, "Found unexpected NUL while %s, offset %"PRId64
496  "at position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".",
497  activity, offset, (nul_found - inp), len);
498  const char *start_excerpt_at, *eos = inp + len;
499  if ((nul_found - inp) >= 16)
500  start_excerpt_at = nul_found - 16;
501  else
502  start_excerpt_at = inp;
503  size_t excerpt_len = MIN(32, eos - start_excerpt_at);
504  char tmp[65];
505  base16_encode(tmp, sizeof(tmp), start_excerpt_at, excerpt_len);
506  log_warn(LD_BUG, " surrounding string: %s", tmp);
507  }
508 }
509 
512 int
514 {
515  struct stat st;
516  char *journal_content;
517  smartlist_t *added;
518  tor_mmap_t *mm;
519  int total = 0;
520 
521  microdesc_cache_clear(cache);
522 
523  cache->is_loaded = 1;
524 
525  mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
526  if (mm) {
527  warn_if_nul_found(mm->data, mm->size, 0, "scanning microdesc cache");
528  added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
529  SAVED_IN_CACHE, 0, -1, NULL);
530  if (added) {
531  total += smartlist_len(added);
532  smartlist_free(added);
533  }
534  }
535 
536  journal_content = read_file_to_str(cache->journal_fname,
537  RFTS_IGNORE_MISSING, &st);
538  if (journal_content) {
539  cache->journal_len = strlen(journal_content);
540  warn_if_nul_found(journal_content, (size_t)st.st_size, 0,
541  "reading microdesc journal");
542  added = microdescs_add_to_cache(cache, journal_content,
543  journal_content+st.st_size,
544  SAVED_IN_JOURNAL, 0, -1, NULL);
545  if (added) {
546  total += smartlist_len(added);
547  smartlist_free(added);
548  }
549  tor_free(journal_content);
550  }
551  log_info(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
552  total);
553 
554  microdesc_cache_rebuild(cache, 0 /* don't force */);
555 
556  return 0;
557 }
558 
561 #define TOLERATE_MICRODESC_AGE (7*24*60*60)
562 
569 void
570 microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
571 {
572  microdesc_t **mdp, *victim;
573  int dropped=0, kept=0;
574  size_t bytes_dropped = 0;
575  time_t now = time(NULL);
576 
577  /* If we don't know a reasonably live consensus, don't believe last_listed
578  * values: we might be starting up after being down for a while. */
579  if (! force &&
580  ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
581  return;
582 
583  if (cutoff <= 0)
584  cutoff = now - TOLERATE_MICRODESC_AGE;
585 
586  for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
587  const int is_old = (*mdp)->last_listed < cutoff;
588  const unsigned held_by_nodes = (*mdp)->held_by_nodes;
589  if (is_old && !held_by_nodes) {
590  ++dropped;
591  victim = *mdp;
592  mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
593  victim->held_in_map = 0;
594  bytes_dropped += victim->bodylen;
595  microdesc_free(victim);
596  } else {
597  if (is_old) {
598  /* It's old, but it has held_by_nodes set. That's not okay. */
599  /* Let's try to diagnose and fix #7164 . */
601  const networkstatus_t *ns = networkstatus_get_latest_consensus();
602  long networkstatus_age = -1;
603  const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
604  if (ns) {
605  networkstatus_age = now - ns->valid_after;
606  }
607  log_warn(LD_BUG, "Microdescriptor seemed very old "
608  "(last listed %d hours ago vs %d hour cutoff), but is still "
609  "marked as being held by %d node(s). I found %d node(s) "
610  "holding it. Current networkstatus is %ld hours old. "
611  "Hashtable badness is %d.",
612  (int)((now - (*mdp)->last_listed) / 3600),
613  (int)((now - cutoff) / 3600),
614  held_by_nodes,
615  smartlist_len(nodes),
616  networkstatus_age / 3600,
617  ht_badness);
618 
619  SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
620  const char *rs_match = "No RS";
621  const char *rs_present = "";
622  if (node->rs) {
623  if (tor_memeq(node->rs->descriptor_digest,
624  (*mdp)->digest, DIGEST256_LEN)) {
625  rs_match = "Microdesc digest in RS matches";
626  } else {
627  rs_match = "Microdesc digest in RS does match";
628  }
629  if (ns) {
630  /* This should be impossible, but let's see! */
631  rs_present = " RS not present in networkstatus.";
633  if (rs == node->rs) {
634  rs_present = " RS okay in networkstatus.";
635  }
636  });
637  }
638  }
639  log_warn(LD_BUG, " [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s",
640  node_sl_idx,
641  hex_str(node->identity, DIGEST_LEN),
642  node->md, node->rs, node->ri, rs_match, rs_present);
643  } SMARTLIST_FOREACH_END(node);
644  smartlist_free(nodes);
645  (*mdp)->last_listed = now;
646  }
647 
648  ++kept;
649  mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
650  }
651  }
652 
653  if (dropped) {
654  log_info(LD_DIR, "Removed %d/%d microdescriptors as old.",
655  dropped,dropped+kept);
656  cache->bytes_dropped += bytes_dropped;
657  }
658 }
659 
660 static int
661 should_rebuild_md_cache(microdesc_cache_t *cache)
662 {
663  const size_t old_len =
664  cache->cache_content ? cache->cache_content->size : 0;
665  const size_t journal_len = cache->journal_len;
666  const size_t dropped = cache->bytes_dropped;
667 
668  if (journal_len < 16384)
669  return 0; /* Don't bother, not enough has happened yet. */
670  if (dropped > (journal_len + old_len) / 3)
671  return 1; /* We could save 1/3 or more of the currently used space. */
672  if (journal_len > old_len / 2)
673  return 1; /* We should append to the regular file */
674 
675  return 0;
676 }
677 
682 static void
684 {
685  if (!md)
686  return;
687 
688  if (md->saved_location != SAVED_IN_CACHE)
689  tor_free(md->body);
690 
691  md->off = 0;
693  md->body = NULL;
694  md->bodylen = 0;
695  md->no_save = 1;
696 }
697 
702 int
704 {
705  open_file_t *open_file;
706  int fd = -1, res;
707  microdesc_t **mdp;
708  smartlist_t *wrote;
709  ssize_t size;
710  off_t off = 0, off_real;
711  int orig_size, new_size;
712 
713  if (cache == NULL) {
714  cache = the_microdesc_cache;
715  if (cache == NULL)
716  return 0;
717  }
718 
719  /* Remove dead descriptors */
720  microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);
721 
722  if (!force && !should_rebuild_md_cache(cache))
723  return 0;
724 
725  log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
726 
727  orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
728  orig_size += (int)cache->journal_len;
729 
730  fd = start_writing_to_file(cache->cache_fname,
731  OPEN_FLAGS_REPLACE|O_BINARY,
732  0600, &open_file);
733  if (fd < 0)
734  return -1;
735 
736  wrote = smartlist_new();
737 
738  HT_FOREACH(mdp, microdesc_map, &cache->map) {
739  microdesc_t *md = *mdp;
740  size_t annotation_len;
741  if (md->no_save || !md->body)
742  continue;
743 
744  size = dump_microdescriptor(fd, md, &annotation_len);
745  if (size < 0) {
747 
748  /* rewind, in case it was a partial write. */
749  tor_fd_setpos(fd, off);
750  continue;
751  }
752  tor_assert(((size_t)size) == annotation_len + md->bodylen);
753  md->off = off + annotation_len;
754  off += size;
755  off_real = tor_fd_getpos(fd);
756  if (off_real != off) {
757  log_warn(LD_BUG, "Discontinuity in position in microdescriptor cache."
758  "By my count, I'm at %"PRId64
759  ", but I should be at %"PRId64,
760  (int64_t)(off), (int64_t)(off_real));
761  if (off_real >= 0)
762  off = off_real;
763  }
764  if (md->saved_location != SAVED_IN_CACHE) {
765  tor_free(md->body);
767  }
768  smartlist_add(wrote, md);
769  }
770 
771  /* We must do this unmap _before_ we call finish_writing_to_file(), or
772  * windows will not actually replace the file. */
773  if (cache->cache_content) {
774  res = tor_munmap_file(cache->cache_content);
775  if (res != 0) {
776  log_warn(LD_FS,
777  "Failed to unmap old microdescriptor cache while rebuilding");
778  }
779  cache->cache_content = NULL;
780  }
781 
782  if (finish_writing_to_file(open_file) < 0) {
783  log_warn(LD_DIR, "Error rebuilding microdescriptor cache: %s",
784  strerror(errno));
785  /* Okay. Let's prevent from making things worse elsewhere. */
786  cache->cache_content = NULL;
787  HT_FOREACH(mdp, microdesc_map, &cache->map) {
788  microdesc_t *md = *mdp;
789  if (md->saved_location == SAVED_IN_CACHE) {
791  }
792  }
793  smartlist_free(wrote);
794  return -1;
795  }
796 
797  cache->cache_content = tor_mmap_file(cache->cache_fname);
798 
799  if (!cache->cache_content && smartlist_len(wrote)) {
800  log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
801  cache->cache_fname);
802  smartlist_free(wrote);
803  return -1;
804  }
805  SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
806  tor_assert(md->saved_location == SAVED_IN_CACHE);
807  md->body = (char*)cache->cache_content->data + md->off;
808  if (PREDICT_UNLIKELY(
809  md->bodylen < 9 || fast_memneq(md->body, "onion-key", 9) != 0)) {
810  /* XXXX once bug 2022 is solved, we can kill this block and turn it
811  * into just the tor_assert(fast_memeq) */
812  off_t avail = cache->cache_content->size - md->off;
813  char *bad_str;
814  tor_assert(avail >= 0);
815  bad_str = tor_strndup(md->body, MIN(128, (size_t)avail));
816  log_err(LD_BUG, "After rebuilding microdesc cache, offsets seem wrong. "
817  " At offset %d, I expected to find a microdescriptor starting "
818  " with \"onion-key\". Instead I got %s.",
819  (int)md->off, escaped(bad_str));
820  tor_free(bad_str);
821  tor_assert(fast_memeq(md->body, "onion-key", 9));
822  }
823  } SMARTLIST_FOREACH_END(md);
824 
825  smartlist_free(wrote);
826 
827  write_str_to_file(cache->journal_fname, "", 1);
828  cache->journal_len = 0;
829  cache->bytes_dropped = 0;
830 
831  new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
832  log_info(LD_DIR, "Done rebuilding microdesc cache. "
833  "Saved %d bytes; %d still used.",
834  orig_size-new_size, new_size);
835 
836  return 0;
837 }
838 
841 void
843 {
844  microdesc_t **mdp;
845  if (!the_microdesc_cache)
846  return;
847 
848  HT_FOREACH(mdp, microdesc_map, &the_microdesc_cache->map) {
849  microdesc_t *md = *mdp;
850  unsigned int found=0;
851  const smartlist_t *nodes = nodelist_get_list();
852  SMARTLIST_FOREACH(nodes, node_t *, node, {
853  if (node->md == md) {
854  ++found;
855  }
856  });
857  tor_assert(found == md->held_by_nodes);
858  }
859 }
860 
863 void
864 microdesc_free_(microdesc_t *md, const char *fname, int lineno)
865 {
866  if (!md)
867  return;
868 
869  /* Make sure that the microdesc was really removed from the appropriate data
870  structures. */
871  if (md->held_in_map) {
873  microdesc_t *md2 = HT_FIND(microdesc_map, &cache->map, md);
874  if (md2 == md) {
875  log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
876  "in microdesc_map", fname, lineno);
877  HT_REMOVE(microdesc_map, &cache->map, md);
878  } else {
879  log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_in_map "
880  "set, but microdesc was not in the map.", fname, lineno);
881  }
883  }
884  if (md->held_by_nodes) {
886  int found=0;
887  const smartlist_t *nodes = nodelist_get_list();
888  const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
889  SMARTLIST_FOREACH(nodes, node_t *, node, {
890  if (node->md == md) {
891  ++found;
892  node->md = NULL;
893  }
894  });
895  if (found) {
896  log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
897  "referenced %d node(s); held_by_nodes == %u, ht_badness == %d",
898  fname, lineno, found, md->held_by_nodes, ht_badness);
899  } else {
900  log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_by_nodes "
901  "set to %u, but md was not referenced by any nodes. "
902  "ht_badness == %d",
903  fname, lineno, md->held_by_nodes, ht_badness);
904  }
906  }
907  //tor_assert(md->held_in_map == 0);
908  //tor_assert(md->held_by_nodes == 0);
909 
910  if (md->onion_pkey)
911  tor_free(md->onion_pkey);
914  if (md->body && md->saved_location != SAVED_IN_CACHE)
915  tor_free(md->body);
916 
917  nodefamily_free(md->family);
918  short_policy_free(md->exit_policy);
919  short_policy_free(md->ipv6_exit_policy);
920 
921  tor_free(md);
922 }
923 
925 void
927 {
928  if (the_microdesc_cache) {
930  tor_free(the_microdesc_cache->cache_fname);
931  tor_free(the_microdesc_cache->journal_fname);
933  }
934 
935  if (outdated_dirserver_list) {
936  SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
937  smartlist_free(outdated_dirserver_list);
938  }
939 }
940 
943 microdesc_t *
945 {
946  microdesc_t *md, search;
947  if (!cache)
948  cache = get_microdesc_cache();
949  memcpy(search.digest, d, DIGEST256_LEN);
950  md = HT_FIND(microdesc_map, &cache->map, &search);
951  return md;
952 }
953 
958 smartlist_t *
960  int downloadable_only, digest256map_t *skip)
961 {
962  smartlist_t *result = smartlist_new();
963  time_t now = time(NULL);
964  tor_assert(ns->flavor == FLAV_MICRODESC);
966  if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
967  continue;
968  if (downloadable_only &&
969  !download_status_is_ready(&rs->dl_status, now))
970  continue;
971  if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
972  continue;
973  if (fast_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
974  continue;
975  /* XXXX Also skip if we're a noncache and wouldn't use this router.
976  * XXXX NM Microdesc
977  */
978  smartlist_add(result, rs->descriptor_digest);
979  } SMARTLIST_FOREACH_END(rs);
980  return result;
981 }
982 
990 void
992 {
993  const or_options_t *options = get_options();
994  networkstatus_t *consensus;
995  smartlist_t *missing;
996  digest256map_t *pending;
997 
998  if (should_delay_dir_fetches(options, NULL))
999  return;
1000  if (directory_too_idle_to_fetch_descriptors(options, now))
1001  return;
1002 
1003  /* Give up if we don't have a reasonably live consensus. */
1004  consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
1005  if (!consensus)
1006  return;
1007 
1008  if (!we_fetch_microdescriptors(options))
1009  return;
1010 
1011  pending = digest256map_new();
1013 
1014  missing = microdesc_list_missing_digest256(consensus,
1016  1,
1017  pending);
1018  digest256map_free(pending, NULL);
1019 
1021  missing, NULL, now);
1022 
1023  smartlist_free(missing);
1024 }
1025 
1030 void
1032 {
1034  microdesc_t *md;
1035  networkstatus_t *ns =
1036  networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
1037 
1038  if (! ns)
1039  return;
1040 
1041  tor_assert(ns->flavor == FLAV_MICRODESC);
1042 
1044  md = microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest);
1045  if (md && ns->valid_after > md->last_listed)
1046  md->last_listed = ns->valid_after;
1047  } SMARTLIST_FOREACH_END(rs);
1048 }
1049 
1052 int
1054 {
1055  if (options->UseMicrodescriptors == 0)
1056  return 0; /* the user explicitly picked no */
1057  return 1; /* yes and auto both mean yes */
1058 }
1059 
1061 int
1063 {
1064  if (directory_caches_dir_info(options))
1065  return 1;
1066  if (options->FetchUselessDescriptors)
1067  return 1;
1068  return we_use_microdescriptors_for_circuits(options);
1069 }
1070 
1072 int
1074 {
1075  if (directory_caches_dir_info(options))
1076  return 1;
1077  if (options->FetchUselessDescriptors)
1078  return 1;
1079  return ! we_use_microdescriptors_for_circuits(options);
1080 }
1081 
1084 usable_consensus_flavor,(void))
1085 {
1086  if (we_use_microdescriptors_for_circuits(get_options())) {
1087  return FLAV_MICRODESC;
1088  } else {
1089  return FLAV_NS;
1090  }
1091 }
Header file for dirserv.c.
void microdesc_free_(microdesc_t *md, const char *fname, int lineno)
Definition: microdesc.c:864
struct short_policy_t * exit_policy
Definition: microdesc_st.h:76
smartlist_t * nodelist_find_nodes_with_microdesc(const microdesc_t *md)
Definition: nodelist.c:767
Header file for circuitbuild.c.
Definition: node_st.h:28
int tor_fd_setpos(int fd, off_t pos)
Definition: fdio.c:79
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
HT_PROTOTYPE(HT_GENERATE2(microdesc_map, HT_GENERATE2(microdesc_t, HT_GENERATE2(node, HT_GENERATE2(microdesc_hash_, HT_GENERATE2(microdesc_eq_)
Definition: microdesc.c:91
Header file for nodefamily.c.
void router_dir_info_changed(void)
Definition: nodelist.c:2330
int we_use_microdescriptors_for_circuits(const or_options_t *options)
Definition: microdesc.c:1053
static ssize_t dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
Definition: microdesc.c:201
void smartlist_add_strdup(struct smartlist_t *sl, const char *string)
void microdesc_check_counts(void)
Definition: microdesc.c:842
#define LD_GENERAL
Definition: log.h:59
void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
Definition: microdesc.c:570
Header file for nodelist.c.
Header for fdio.c.
Header file for directory.c.
void smartlist_add(smartlist_t *sl, void *element)
ssize_t write_all_to_fd(int fd, const char *buf, size_t count)
Definition: files.c:162
void list_pending_microdesc_downloads(digest256map_t *result)
Definition: routerlist.c:2309
Header file for config.c.
int fast_mem_is_zero(const char *mem, size_t len)
Definition: util_string.c:74
smartlist_t * microdescs_add_to_cache(microdesc_cache_t *cache, const char *s, const char *eos, saved_location_t where, int no_save, time_t listed_at, smartlist_t *requested_digests256)
Definition: microdesc.c:291
off_t tor_fd_getpos(int fd)
Definition: fdio.c:47
Header file for microdesc.c.
int we_fetch_router_descriptors(const or_options_t *options)
Definition: microdesc.c:1073
int UseMicrodescriptors
int directory_too_idle_to_fetch_descriptors(const or_options_t *options, time_t now)
Definition: dirserv.c:169
#define tor_free(p)
Definition: malloc.h:52
#define tor_fragile_assert()
Definition: util_bug.h:241
void update_microdescs_from_networkstatus(time_t now)
Definition: microdesc.c:1031
#define SMARTLIST_DEL_CURRENT(sl, var)
size_t size
Definition: mmap.h:26
Header file for microdesc_parse.c.
int download_status_is_ready(download_status_t *dls, time_t now)
Definition: dlstatus.c:376
dir_server_t * router_get_trusteddirserver_by_digest(const char *digest)
Definition: dirlist.c:112
void * tor_reallocarray_(void *ptr, size_t sz1, size_t sz2)
Definition: malloc.c:146
void update_microdesc_downloads(time_t now)
Definition: microdesc.c:991
saved_location_t
Definition: or.h:746
size_t bodylen
Definition: microdesc_st.h:51
void microdesc_reset_outdated_dirservers_list(void)
Definition: microdesc.c:184
time_t last_listed
Definition: microdesc_st.h:29
#define DIGEST256_LEN
Definition: digest_sizes.h:23
Header file for policies.c.
tor_assert(buffer)
node_t * nodelist_add_microdesc(microdesc_t *md)
Definition: nodelist.c:549
smartlist_t * microdescs_parse_from_string(const char *s, const char *eos, int allow_annotations, saved_location_t where, smartlist_t *invalid_digests_out)
static int microdesc_eq_(microdesc_t *a, microdesc_t *b)
Definition: microdesc.c:86
int finish_writing_to_file(open_file_t *file_data)
Definition: files.c:449
int tor_memeq(const void *a, const void *b, size_t sz)
Definition: di_ops.c:107
unsigned int held_by_nodes
Definition: microdesc_st.h:37
void download_status_mark_impossible(download_status_t *dl)
Definition: dlstatus.c:388
#define DIGEST_LEN
Definition: digest_sizes.h:20
Master header file for Tor-specific functionality.
const char * hex_str(const char *from, size_t fromlen)
Definition: binascii.c:34
const char * data
Definition: mmap.h:25
Header file for circuitbuild.c.
#define TOLERATE_MICRODESC_AGE
Definition: microdesc.c:561
void microdesc_cache_clear(microdesc_cache_t *cache)
Definition: microdesc.c:463
int microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
Definition: microdesc.c:703
void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:478
struct ed25519_public_key_t * ed25519_identity_pkey
Definition: microdesc_st.h:68
void tor_free_(void *mem)
Definition: malloc.c:227
char * body
Definition: microdesc_st.h:49
unsigned int held_in_map
Definition: microdesc_st.h:35
#define RFTS_IGNORE_MISSING
Definition: files.h:97
static microdesc_cache_t * the_microdesc_cache
Definition: microdesc.c:245
int start_writing_to_file(const char *fname, int open_flags, int mode, open_file_t **data_out)
Definition: files.c:301
int microdesc_relay_is_outdated_dirserver(const char *relay_digest)
Definition: microdesc.c:161
static void microdesc_wipe_body(microdesc_t *md)
Definition: microdesc.c:683
char * onion_pkey
Definition: microdesc_st.h:61
MOCK_IMPL(int, usable_consensus_flavor,(void))
Definition: microdesc.c:1083
microdesc_t * microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
Definition: microdesc.c:944
int FetchUselessDescriptors
#define LD_FS
Definition: log.h:67
smartlist_t * routerstatus_list
static microdesc_cache_t * get_microdesc_cache_noload(void)
Definition: microdesc.c:261
char descriptor_digest[DIGEST256_LEN]
#define HEX_DIGEST_LEN
Definition: crypto_digest.h:35
#define LD_DIR
Definition: log.h:85
consensus_flavor_t flavor
void launch_descriptor_downloads(int purpose, smartlist_t *downloadable, const routerstatus_t *source, time_t now)
Definition: routerlist.c:2433
int smartlist_contains_string(const smartlist_t *sl, const char *element)
Definition: smartlist.c:93
int tor_snprintf(char *str, size_t size, const char *format,...)
Definition: printf.c:27
void format_iso_time(char *buf, time_t t)
Definition: time_fmt.c:295
microdesc_cache_t * get_microdesc_cache(void)
Definition: microdesc.c:249
struct curve25519_public_key_t * onion_curve25519_pkey
Definition: microdesc_st.h:66
#define SMARTLIST_FOREACH(sl, type, var, cmd)
void microdesc_free_all(void)
Definition: microdesc.c:926
Header file for router.c.
const char * escaped(const char *s)
Definition: escape.c:126
static unsigned int microdesc_hash_(microdesc_t *md)
Definition: microdesc.c:78
smartlist_t * microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, int downloadable_only, digest256map_t *skip)
Definition: microdesc.c:959
struct short_policy_t * ipv6_exit_policy
Definition: microdesc_st.h:78
Header file for dlstatus.c.
#define log_fn(severity, domain, args,...)
Definition: log.h:272
#define DIR_PURPOSE_FETCH_MICRODESC
Definition: directory.h:70
char digest[DIGEST256_LEN]
Definition: microdesc_st.h:53
time_t approx_time(void)
Definition: approx_time.c:32
Header file for dirlist.c.
int abort_writing_to_file(open_file_t *file_data)
Definition: files.c:457
int directory_caches_dir_info(const or_options_t *options)
Definition: dirserv.c:143
int we_fetch_microdescriptors(const or_options_t *options)
Definition: microdesc.c:1062
int microdesc_cache_reload(microdesc_cache_t *cache)
Definition: microdesc.c:513
unsigned int no_save
Definition: microdesc_st.h:33
struct nodefamily_t * family
Definition: microdesc_st.h:74
smartlist_t * microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, int no_save)
Definition: microdesc.c:381
void smartlist_clear(smartlist_t *sl)
saved_location_bitfield_t saved_location
Definition: microdesc_st.h:31
int should_delay_dir_fetches(const or_options_t *options, const char **msg_out)
Header file for networkstatus.c.
#define LD_BUG
Definition: log.h:83
Header file for routerlist.c.