Tor  0.4.7.0-alpha-dev
microdesc.c
Go to the documentation of this file.
1 /* Copyright (c) 2009-2021, The Tor Project, Inc. */
2 /* See LICENSE for licensing information */
3 
4 /**
5  * \file microdesc.c
6  *
7  * \brief Implements microdescriptors -- an abbreviated description of
8  * less-frequently-changing router information.
9  */
10 
11 #include "core/or/or.h"
12 
13 #include "lib/fdio/fdio.h"
14 
15 #include "app/config/config.h"
16 #include "core/or/circuitbuild.h"
17 #include "core/or/policies.h"
30 #include "feature/relay/router.h"
31 
36 
37 #ifdef HAVE_FCNTL_H
38 #include <fcntl.h>
39 #endif
40 #ifdef HAVE_SYS_STAT_H
41 #include <sys/stat.h>
42 #endif
43 
44 /** A data structure to hold a bunch of cached microdescriptors. There are
45  * two active files in the cache: a "cache file" that we mmap, and a "journal
46  * file" that we append to. Periodically, we rebuild the cache file to hold
47  * only the microdescriptors that we want to keep */
49  /** Map from sha256-digest to microdesc_t for every microdesc_t in the
50  * cache. */
51  HT_HEAD(microdesc_map, microdesc_t) map;
52 
53  /** Name of the cache file. */
54  char *cache_fname;
55  /** Name of the journal file. */
56  char *journal_fname;
57  /** Mmap'd contents of the cache file, or NULL if there is none. */
58  tor_mmap_t *cache_content;
59  /** Number of bytes used in the journal file. */
60  size_t journal_len;
61  /** Number of bytes in descriptors removed as too old. */
62  size_t bytes_dropped;
63 
64  /** Total bytes of microdescriptor bodies we have added to this cache */
65  uint64_t total_len_seen;
66  /** Total number of microdescriptors we have added to this cache */
67  unsigned n_seen;
68 
69  /** True iff we have loaded this cache from disk ever. */
70  int is_loaded;
71 };
72 
74 static void warn_if_nul_found(const char *inp, size_t len, int64_t offset,
75  const char *activity);
76 
77 /** Helper: computes a hash of <b>md</b> to place it in a hash table. */
78 static inline unsigned int
80 {
81  return (unsigned) siphash24g(md->digest, sizeof(md->digest));
82 }
83 
84 /** Helper: compares <b>a</b> and <b>b</b> for equality for hash-table
85  * purposes. */
86 static inline int
88 {
89  return tor_memeq(a->digest, b->digest, DIGEST256_LEN);
90 }
91 
92 HT_PROTOTYPE(microdesc_map, microdesc_t, node,
94 HT_GENERATE2(microdesc_map, microdesc_t, node,
97 
98 /************************* md fetch fail cache *****************************/
99 
100 /* If we end up with too many outdated dirservers, something probably went
101  * wrong so clean up the list. */
102 #define TOO_MANY_OUTDATED_DIRSERVERS 30
103 
104 /** List of dirservers with outdated microdesc information. The smartlist is
105  * filled with the hex digests of outdated dirservers. */
107 
108 /** Note that we failed to fetch a microdescriptor from the relay with
109  * <b>relay_digest</b> (of size DIGEST_LEN). */
110 void
111 microdesc_note_outdated_dirserver(const char *relay_digest)
112 {
113  char relay_hexdigest[HEX_DIGEST_LEN+1];
114 
115  /* If we have a reasonably live consensus, then most of our dirservers should
116  * still be caching all the microdescriptors in it. Reasonably live
117  * consensuses are up to a day old (or a day in the future). But
118  * microdescriptors expire 7 days after the last consensus that referenced
119  * them. */
121  FLAV_MICRODESC)) {
122  return;
123  }
124 
127  }
128 
130 
131  /* If the list grows too big, clean it up */
132  if (smartlist_len(outdated_dirserver_list) > TOO_MANY_OUTDATED_DIRSERVERS) {
133  log_info(LD_GENERAL,"Too many outdated directory servers (%d). Resetting.",
134  smartlist_len(outdated_dirserver_list));
136  }
137 
138  /* Turn the binary relay digest to a hex since smartlists have better support
139  * for strings than digests. */
140  base16_encode(relay_hexdigest,sizeof(relay_hexdigest),
141  relay_digest, DIGEST_LEN);
142 
143  /* Make sure we don't add a dirauth as an outdated dirserver */
144  if (router_get_trusteddirserver_by_digest(relay_digest)) {
145  log_info(LD_GENERAL, "Auth %s gave us outdated dirinfo.", relay_hexdigest);
146  return;
147  }
148 
149  /* Don't double-add outdated dirservers */
150  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
151  return;
152  }
153 
154  /* Add it to the list of outdated dirservers */
156 
157  log_info(LD_GENERAL, "Noted %s as outdated md dirserver", relay_hexdigest);
158 }
159 
160 /** Return True if the relay with <b>relay_digest</b> (size DIGEST_LEN) is an
161  * outdated dirserver */
162 int
163 microdesc_relay_is_outdated_dirserver(const char *relay_digest)
164 {
165  char relay_hexdigest[HEX_DIGEST_LEN+1];
166 
168  return 0;
169  }
170 
171  /* Convert identity digest to hex digest */
172  base16_encode(relay_hexdigest, sizeof(relay_hexdigest),
173  relay_digest, DIGEST_LEN);
174 
175  /* Last time we tried to fetch microdescs, was this directory mirror missing
176  * any mds we asked for? */
177  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
178  return 1;
179  }
180 
181  return 0;
182 }
183 
184 /** Reset the list of outdated dirservers. */
185 void
187 {
189  return;
190  }
191 
194 }
195 
196 /****************************************************************************/
197 
198 /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
199  * On success, return the total number of bytes written, and set
200  * *<b>annotation_len_out</b> to the number of bytes written as
201  * annotations. */
202 static ssize_t
203 dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
204 {
205  ssize_t r = 0;
206  ssize_t written;
207  if (md->body == NULL) {
208  *annotation_len_out = 0;
209  return 0;
210  }
211  /* XXXX drops unknown annotations. */
212  if (md->last_listed) {
213  char buf[ISO_TIME_LEN+1];
214  char annotation[ISO_TIME_LEN+32];
215  format_iso_time(buf, md->last_listed);
216  tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
217  if (write_all_to_fd(fd, annotation, strlen(annotation)) < 0) {
218  log_warn(LD_DIR,
219  "Couldn't write microdescriptor annotation: %s",
220  strerror(errno));
221  return -1;
222  }
223  r += strlen(annotation);
224  *annotation_len_out = r;
225  } else {
226  *annotation_len_out = 0;
227  }
228 
229  md->off = tor_fd_getpos(fd);
230  warn_if_nul_found(md->body, md->bodylen, (int64_t) md->off,
231  "dumping a microdescriptor");
232  written = write_all_to_fd(fd, md->body, md->bodylen);
233  if (written != (ssize_t)md->bodylen) {
234  written = written < 0 ? 0 : written;
235  log_warn(LD_DIR,
236  "Couldn't dump microdescriptor (wrote %ld out of %lu): %s",
237  (long)written, (unsigned long)md->bodylen,
238  strerror(errno));
239  return -1;
240  }
241  r += md->bodylen;
242  return r;
243 }
244 
245 /** Holds a pointer to the current microdesc_cache_t object, or NULL if no
246  * such object has been allocated. */
248 
249 /** Return a pointer to the microdescriptor cache, loading it if necessary. */
252 {
254  if (PREDICT_UNLIKELY(cache->is_loaded == 0)) {
255  microdesc_cache_reload(cache);
256  }
257  return cache;
258 }
259 
260 /** Return a pointer to the microdescriptor cache, creating (but not loading)
261  * it if necessary. */
262 static microdesc_cache_t *
264 {
265  if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
266  microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache));
267  HT_INIT(microdesc_map, &cache->map);
268  cache->cache_fname = get_cachedir_fname("cached-microdescs");
269  cache->journal_fname = get_cachedir_fname("cached-microdescs.new");
270  the_microdesc_cache = cache;
271  }
272  return the_microdesc_cache;
273 }
274 
275 /* There are three sources of microdescriptors:
276  1) Generated by us while acting as a directory authority.
277  2) Loaded from the cache on disk.
278  3) Downloaded.
279 */
280 
281 /** Decode the microdescriptors from the string starting at <b>s</b> and
282  * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no_save</b>,
283  * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE,
284  * leave their bodies as pointers to the mmap'd cache. If where is
285  * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is not -1,
286  * set the last_listed field of every microdesc to listed_at. If
287  * requested_digests is non-null, then it contains a list of digests we mean
288  * to allow, so we should reject any non-requested microdesc with a different
289  * digest, and alter the list to contain only the digests of those microdescs
290  * we didn't find.
291  * Return a newly allocated list of the added microdescriptors, or NULL */
292 smartlist_t *
294  const char *s, const char *eos, saved_location_t where,
295  int no_save, time_t listed_at,
296  smartlist_t *requested_digests256)
297 {
298  void * const DIGEST_REQUESTED = (void*)1;
299  void * const DIGEST_RECEIVED = (void*)2;
300  void * const DIGEST_INVALID = (void*)3;
301 
302  smartlist_t *descriptors, *added;
303  const int allow_annotations = (where != SAVED_NOWHERE);
304  smartlist_t *invalid_digests = smartlist_new();
305 
306  descriptors = microdescs_parse_from_string(s, eos,
307  allow_annotations,
308  where, invalid_digests);
309  if (listed_at != (time_t)-1) {
310  SMARTLIST_FOREACH(descriptors, microdesc_t *, md,
311  md->last_listed = listed_at);
312  }
313  if (requested_digests256) {
314  digest256map_t *requested;
315  requested = digest256map_new();
316  /* Set requested[d] to DIGEST_REQUESTED for every md we requested. */
317  SMARTLIST_FOREACH(requested_digests256, const uint8_t *, cp,
318  digest256map_set(requested, cp, DIGEST_REQUESTED));
319  /* Set requested[d] to DIGEST_INVALID for every md we requested which we
320  * will never be able to parse. Remove the ones we didn't request from
321  * invalid_digests.
322  */
323  SMARTLIST_FOREACH_BEGIN(invalid_digests, uint8_t *, cp) {
324  if (digest256map_get(requested, cp)) {
325  digest256map_set(requested, cp, DIGEST_INVALID);
326  } else {
327  tor_free(cp);
328  SMARTLIST_DEL_CURRENT(invalid_digests, cp);
329  }
330  } SMARTLIST_FOREACH_END(cp);
331  /* Update requested[d] to 2 for the mds we asked for and got. Delete the
332  * ones we never requested from the 'descriptors' smartlist.
333  */
334  SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
335  if (digest256map_get(requested, (const uint8_t*)md->digest)) {
336  digest256map_set(requested, (const uint8_t*)md->digest,
337  DIGEST_RECEIVED);
338  } else {
339  log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microdesc");
340  microdesc_free(md);
341  SMARTLIST_DEL_CURRENT(descriptors, md);
342  }
343  } SMARTLIST_FOREACH_END(md);
344  /* Remove the ones we got or the invalid ones from requested_digests256.
345  */
346  SMARTLIST_FOREACH_BEGIN(requested_digests256, uint8_t *, cp) {
347  void *status = digest256map_get(requested, cp);
348  if (status == DIGEST_RECEIVED || status == DIGEST_INVALID) {
349  tor_free(cp);
350  SMARTLIST_DEL_CURRENT(requested_digests256, cp);
351  }
352  } SMARTLIST_FOREACH_END(cp);
353  digest256map_free(requested, NULL);
354  }
355 
356  /* For every requested microdescriptor that was unparseable, mark it
357  * as not to be retried. */
358  if (smartlist_len(invalid_digests)) {
359  networkstatus_t *ns =
361  if (ns) {
362  SMARTLIST_FOREACH_BEGIN(invalid_digests, char *, d) {
363  routerstatus_t *rs =
365  if (rs && tor_memeq(d, rs->descriptor_digest, DIGEST256_LEN)) {
366  download_status_mark_impossible(&rs->dl_status);
367  }
368  } SMARTLIST_FOREACH_END(d);
369  }
370  }
371  SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
372  smartlist_free(invalid_digests);
373 
374  added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
375  smartlist_free(descriptors);
376  return added;
377 }
378 
379 /** As microdescs_add_to_cache, but takes a list of microdescriptors instead of
380  * a string to decode. Frees any members of <b>descriptors</b> that it does
381  * not add. */
382 smartlist_t *
384  smartlist_t *descriptors, saved_location_t where,
385  int no_save)
386 {
387  smartlist_t *added;
388  open_file_t *open_file = NULL;
389  int fd = -1;
390  // int n_added = 0;
391  ssize_t size = 0;
392 
393  if (where == SAVED_NOWHERE && !no_save) {
394  fd = start_writing_to_file(cache->journal_fname,
395  OPEN_FLAGS_APPEND|O_BINARY,
396  0600, &open_file);
397  if (fd < 0) {
398  log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
399  cache->journal_fname, strerror(errno));
400  }
401  }
402 
403  added = smartlist_new();
404  SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
405  microdesc_t *md2;
406  md2 = HT_FIND(microdesc_map, &cache->map, md);
407  if (md2) {
408  /* We already had this one. */
409  if (md2->last_listed < md->last_listed)
410  md2->last_listed = md->last_listed;
411  microdesc_free(md);
412  if (where != SAVED_NOWHERE)
413  cache->bytes_dropped += size;
414  continue;
415  }
416 
417  /* Okay, it's a new one. */
418  if (fd >= 0) {
419  size_t annotation_len;
420  size = dump_microdescriptor(fd, md, &annotation_len);
421  if (size < 0) {
422  /* we already warned in dump_microdescriptor */
423  abort_writing_to_file(open_file);
424  fd = -1;
425  } else {
426  md->saved_location = SAVED_IN_JOURNAL;
427  cache->journal_len += size;
428  }
429  } else {
430  md->saved_location = where;
431  }
432 
433  md->no_save = no_save;
434 
435  HT_INSERT(microdesc_map, &cache->map, md);
436  md->held_in_map = 1;
437  smartlist_add(added, md);
438  ++cache->n_seen;
439  cache->total_len_seen += md->bodylen;
440  } SMARTLIST_FOREACH_END(md);
441 
442  if (fd >= 0) {
443  if (finish_writing_to_file(open_file) < 0) {
444  log_warn(LD_DIR, "Error appending to microdescriptor file: %s",
445  strerror(errno));
446  smartlist_clear(added);
447  return added;
448  }
449  }
450 
451  {
453  if (ns && ns->flavor == FLAV_MICRODESC)
455  }
456 
457  if (smartlist_len(added))
459 
460  return added;
461 }
462 
463 /** Remove every microdescriptor in <b>cache</b>. */
464 void
466 {
467  microdesc_t **entry, **next;
468 
469  for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
470  microdesc_t *md = *entry;
471  next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
472  md->held_in_map = 0;
473  microdesc_free(md);
474  }
475  HT_CLEAR(microdesc_map, &cache->map);
476  if (cache->cache_content) {
477  int res = tor_munmap_file(cache->cache_content);
478  if (res != 0) {
479  log_warn(LD_FS,
480  "tor_munmap_file() failed clearing microdesc cache; "
481  "we are probably about to leak memory.");
482  /* TODO something smarter? */
483  }
484  cache->cache_content = NULL;
485  }
486  cache->total_len_seen = 0;
487  cache->n_seen = 0;
488  cache->bytes_dropped = 0;
489 }
490 
491 static void
492 warn_if_nul_found(const char *inp, size_t len, int64_t offset,
493  const char *activity)
494 {
495  const char *nul_found = memchr(inp, 0, len);
496  if (BUG(nul_found)) {
497  log_warn(LD_BUG, "Found unexpected NUL while %s, offset %"PRId64
498  "at position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".",
499  activity, offset, (nul_found - inp), len);
500  const char *start_excerpt_at, *eos = inp + len;
501  if ((nul_found - inp) >= 16)
502  start_excerpt_at = nul_found - 16;
503  else
504  start_excerpt_at = inp;
505  size_t excerpt_len = MIN(32, eos - start_excerpt_at);
506  char tmp[65];
507  base16_encode(tmp, sizeof(tmp), start_excerpt_at, excerpt_len);
508  log_warn(LD_BUG, " surrounding string: %s", tmp);
509  }
510 }
511 
512 /** Reload the contents of <b>cache</b> from disk. If it is empty, load it
513  * for the first time. Return 0 on success, -1 on failure. */
514 int
516 {
517  struct stat st;
518  char *journal_content;
519  smartlist_t *added;
520  tor_mmap_t *mm;
521  int total = 0;
522 
523  microdesc_cache_clear(cache);
524 
525  cache->is_loaded = 1;
526 
527  mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
528  if (mm) {
529  warn_if_nul_found(mm->data, mm->size, 0, "scanning microdesc cache");
530  added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
531  SAVED_IN_CACHE, 0, -1, NULL);
532  if (added) {
533  total += smartlist_len(added);
534  smartlist_free(added);
535  }
536  }
537 
538  journal_content = read_file_to_str(cache->journal_fname,
539  RFTS_IGNORE_MISSING, &st);
540  if (journal_content) {
541  cache->journal_len = strlen(journal_content);
542  warn_if_nul_found(journal_content, (size_t)st.st_size, 0,
543  "reading microdesc journal");
544  added = microdescs_add_to_cache(cache, journal_content,
545  journal_content+st.st_size,
546  SAVED_IN_JOURNAL, 0, -1, NULL);
547  if (added) {
548  total += smartlist_len(added);
549  smartlist_free(added);
550  }
551  tor_free(journal_content);
552  }
553  log_info(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
554  total);
555 
556  microdesc_cache_rebuild(cache, 0 /* don't force */);
557 
558  return 0;
559 }
560 
561 /** By default, we remove any microdescriptors that have gone at least this
562  * long without appearing in a current consensus. */
563 #define TOLERATE_MICRODESC_AGE (7*24*60*60)
564 
565 /** Remove all microdescriptors from <b>cache</b> that haven't been listed for
566  * a long time. Does not rebuild the cache on disk. If <b>cutoff</b> is
567  * positive, specifically remove microdescriptors that have been unlisted
568  * since <b>cutoff</b>. If <b>force</b> is true, remove microdescriptors even
569  * if we have no current live microdescriptor consensus.
570  */
571 void
572 microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
573 {
574  microdesc_t **mdp, *victim;
575  int dropped=0, kept=0;
576  size_t bytes_dropped = 0;
577  time_t now = time(NULL);
578 
579  /* If we don't know a reasonably live consensus, don't believe last_listed
580  * values: we might be starting up after being down for a while. */
581  if (! force &&
582  ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
583  return;
584 
585  if (cutoff <= 0)
586  cutoff = now - TOLERATE_MICRODESC_AGE;
587 
588  for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
589  const int is_old = (*mdp)->last_listed < cutoff;
590  const unsigned held_by_nodes = (*mdp)->held_by_nodes;
591  if (is_old && !held_by_nodes) {
592  ++dropped;
593  victim = *mdp;
594  mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
595  victim->held_in_map = 0;
596  bytes_dropped += victim->bodylen;
597  microdesc_free(victim);
598  } else {
599  if (is_old) {
600  /* It's old, but it has held_by_nodes set. That's not okay. */
601  /* Let's try to diagnose and fix #7164 . */
604  long networkstatus_age = -1;
605  const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
606  if (ns) {
607  networkstatus_age = now - ns->valid_after;
608  }
609  log_warn(LD_BUG, "Microdescriptor seemed very old "
610  "(last listed %d hours ago vs %d hour cutoff), but is still "
611  "marked as being held by %d node(s). I found %d node(s) "
612  "holding it. Current networkstatus is %ld hours old. "
613  "Hashtable badness is %d.",
614  (int)((now - (*mdp)->last_listed) / 3600),
615  (int)((now - cutoff) / 3600),
616  held_by_nodes,
617  smartlist_len(nodes),
618  networkstatus_age / 3600,
619  ht_badness);
620 
621  SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
622  const char *rs_match = "No RS";
623  const char *rs_present = "";
624  if (node->rs) {
625  if (tor_memeq(node->rs->descriptor_digest,
626  (*mdp)->digest, DIGEST256_LEN)) {
627  rs_match = "Microdesc digest in RS matches";
628  } else {
629  rs_match = "Microdesc digest in RS does match";
630  }
631  if (ns) {
632  /* This should be impossible, but let's see! */
633  rs_present = " RS not present in networkstatus.";
635  if (rs == node->rs) {
636  rs_present = " RS okay in networkstatus.";
637  }
638  });
639  }
640  }
641  log_warn(LD_BUG, " [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s",
642  node_sl_idx,
643  hex_str(node->identity, DIGEST_LEN),
644  node->md, node->rs, node->ri, rs_match, rs_present);
645  } SMARTLIST_FOREACH_END(node);
646  smartlist_free(nodes);
647  (*mdp)->last_listed = now;
648  }
649 
650  ++kept;
651  mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
652  }
653  }
654 
655  if (dropped) {
656  log_info(LD_DIR, "Removed %d/%d microdescriptors as old.",
657  dropped,dropped+kept);
658  cache->bytes_dropped += bytes_dropped;
659  }
660 }
661 
662 static int
663 should_rebuild_md_cache(microdesc_cache_t *cache)
664 {
665  const size_t old_len =
666  cache->cache_content ? cache->cache_content->size : 0;
667  const size_t journal_len = cache->journal_len;
668  const size_t dropped = cache->bytes_dropped;
669 
670  if (journal_len < 16384)
671  return 0; /* Don't bother, not enough has happened yet. */
672  if (dropped > (journal_len + old_len) / 3)
673  return 1; /* We could save 1/3 or more of the currently used space. */
674  if (journal_len > old_len / 2)
675  return 1; /* We should append to the regular file */
676 
677  return 0;
678 }
679 
680 /**
681  * Mark <b>md</b> as having no body, and release any storage previously held
682  * by its body.
683  */
684 static void
686 {
687  if (!md)
688  return;
689 
690  if (md->saved_location != SAVED_IN_CACHE)
691  tor_free(md->body);
692 
693  md->off = 0;
695  md->body = NULL;
696  md->bodylen = 0;
697  md->no_save = 1;
698 }
699 
700 /** Regenerate the main cache file for <b>cache</b>, clear the journal file,
701  * and update every microdesc_t in the cache with pointers to its new
702  * location. If <b>force</b> is true, do this unconditionally. If
703  * <b>force</b> is false, do it only if we expect to save space on disk. */
704 int
706 {
707  open_file_t *open_file;
708  int fd = -1, res;
709  microdesc_t **mdp;
710  smartlist_t *wrote;
711  ssize_t size;
712  off_t off = 0, off_real;
713  int orig_size, new_size;
714 
715  if (cache == NULL) {
716  cache = the_microdesc_cache;
717  if (cache == NULL)
718  return 0;
719  }
720 
721  /* Remove dead descriptors */
722  microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);
723 
724  if (!force && !should_rebuild_md_cache(cache))
725  return 0;
726 
727  log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
728 
729  orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
730  orig_size += (int)cache->journal_len;
731 
732  fd = start_writing_to_file(cache->cache_fname,
733  OPEN_FLAGS_REPLACE|O_BINARY,
734  0600, &open_file);
735  if (fd < 0)
736  return -1;
737 
738  wrote = smartlist_new();
739 
740  HT_FOREACH(mdp, microdesc_map, &cache->map) {
741  microdesc_t *md = *mdp;
742  size_t annotation_len;
743  if (md->no_save || !md->body)
744  continue;
745 
746  size = dump_microdescriptor(fd, md, &annotation_len);
747  if (size < 0) {
749 
750  /* rewind, in case it was a partial write. */
751  tor_fd_setpos(fd, off);
752  continue;
753  }
754  tor_assert(((size_t)size) == annotation_len + md->bodylen);
755  md->off = off + annotation_len;
756  off += size;
757  off_real = tor_fd_getpos(fd);
758  if (off_real != off) {
759  log_warn(LD_BUG, "Discontinuity in position in microdescriptor cache."
760  "By my count, I'm at %"PRId64
761  ", but I should be at %"PRId64,
762  (int64_t)(off), (int64_t)(off_real));
763  if (off_real >= 0)
764  off = off_real;
765  }
766  if (md->saved_location != SAVED_IN_CACHE) {
767  tor_free(md->body);
769  }
770  smartlist_add(wrote, md);
771  }
772 
773  /* We must do this unmap _before_ we call finish_writing_to_file(), or
774  * windows will not actually replace the file. */
775  if (cache->cache_content) {
776  res = tor_munmap_file(cache->cache_content);
777  if (res != 0) {
778  log_warn(LD_FS,
779  "Failed to unmap old microdescriptor cache while rebuilding");
780  }
781  cache->cache_content = NULL;
782  }
783 
784  if (finish_writing_to_file(open_file) < 0) {
785  log_warn(LD_DIR, "Error rebuilding microdescriptor cache: %s",
786  strerror(errno));
787  /* Okay. Let's prevent from making things worse elsewhere. */
788  cache->cache_content = NULL;
789  HT_FOREACH(mdp, microdesc_map, &cache->map) {
790  microdesc_t *md = *mdp;
791  if (md->saved_location == SAVED_IN_CACHE) {
793  }
794  }
795  smartlist_free(wrote);
796  return -1;
797  }
798 
799  cache->cache_content = tor_mmap_file(cache->cache_fname);
800 
801  if (!cache->cache_content && smartlist_len(wrote)) {
802  log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
803  cache->cache_fname);
804  smartlist_free(wrote);
805  return -1;
806  }
807  SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
808  tor_assert(md->saved_location == SAVED_IN_CACHE);
809  md->body = (char*)cache->cache_content->data + md->off;
810  if (PREDICT_UNLIKELY(
811  md->bodylen < 9 || fast_memneq(md->body, "onion-key", 9) != 0)) {
812  /* XXXX once bug 2022 is solved, we can kill this block and turn it
813  * into just the tor_assert(fast_memeq) */
814  off_t avail = cache->cache_content->size - md->off;
815  char *bad_str;
816  tor_assert(avail >= 0);
817  bad_str = tor_strndup(md->body, MIN(128, (size_t)avail));
818  log_err(LD_BUG, "After rebuilding microdesc cache, offsets seem wrong. "
819  " At offset %d, I expected to find a microdescriptor starting "
820  " with \"onion-key\". Instead I got %s.",
821  (int)md->off, escaped(bad_str));
822  tor_free(bad_str);
823  tor_assert(fast_memeq(md->body, "onion-key", 9));
824  }
825  } SMARTLIST_FOREACH_END(md);
826 
827  smartlist_free(wrote);
828 
829  write_str_to_file(cache->journal_fname, "", 1);
830  cache->journal_len = 0;
831  cache->bytes_dropped = 0;
832 
833  new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
834  log_info(LD_DIR, "Done rebuilding microdesc cache. "
835  "Saved %d bytes; %d still used.",
836  orig_size-new_size, new_size);
837 
838  return 0;
839 }
840 
841 /** Make sure that the reference count of every microdescriptor in cache is
842  * accurate. */
843 void
845 {
846  microdesc_t **mdp;
847  if (!the_microdesc_cache)
848  return;
849 
850  HT_FOREACH(mdp, microdesc_map, &the_microdesc_cache->map) {
851  microdesc_t *md = *mdp;
852  unsigned int found=0;
853  const smartlist_t *nodes = nodelist_get_list();
854  SMARTLIST_FOREACH(nodes, node_t *, node, {
855  if (node->md == md) {
856  ++found;
857  }
858  });
859  tor_assert(found == md->held_by_nodes);
860  }
861 }
862 
863 /** Deallocate a single microdescriptor. Note: the microdescriptor MUST have
864  * previously been removed from the cache if it had ever been inserted. */
865 void
866 microdesc_free_(microdesc_t *md, const char *fname, int lineno)
867 {
868  if (!md)
869  return;
870 
871  /* Make sure that the microdesc was really removed from the appropriate data
872  structures. */
873  if (md->held_in_map) {
875  microdesc_t *md2 = HT_FIND(microdesc_map, &cache->map, md);
876  if (md2 == md) {
877  log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
878  "in microdesc_map", fname, lineno);
879  HT_REMOVE(microdesc_map, &cache->map, md);
880  } else {
881  log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_in_map "
882  "set, but microdesc was not in the map.", fname, lineno);
883  }
885  }
886  if (md->held_by_nodes) {
888  int found=0;
889  const smartlist_t *nodes = nodelist_get_list();
890  const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
891  SMARTLIST_FOREACH(nodes, node_t *, node, {
892  if (node->md == md) {
893  ++found;
894  node->md = NULL;
895  }
896  });
897  if (found) {
898  log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
899  "referenced %d node(s); held_by_nodes == %u, ht_badness == %d",
900  fname, lineno, found, md->held_by_nodes, ht_badness);
901  } else {
902  log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_by_nodes "
903  "set to %u, but md was not referenced by any nodes. "
904  "ht_badness == %d",
905  fname, lineno, md->held_by_nodes, ht_badness);
906  }
908  }
909  //tor_assert(md->held_in_map == 0);
910  //tor_assert(md->held_by_nodes == 0);
911 
912  if (md->onion_pkey)
913  tor_free(md->onion_pkey);
916  if (md->body && md->saved_location != SAVED_IN_CACHE)
917  tor_free(md->body);
918 
919  nodefamily_free(md->family);
920  short_policy_free(md->exit_policy);
921  short_policy_free(md->ipv6_exit_policy);
922 
923  tor_free(md);
924 }
925 
926 /** Free all storage held in the microdesc.c module. */
927 void
929 {
930  if (the_microdesc_cache) {
932  tor_free(the_microdesc_cache->cache_fname);
933  tor_free(the_microdesc_cache->journal_fname);
935  }
936 
939  smartlist_free(outdated_dirserver_list);
940  }
941 }
942 
943 /** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
944  * <b>d</b>, return it. Otherwise return NULL. */
945 microdesc_t *
947 {
948  microdesc_t *md, search;
949  if (!cache)
950  cache = get_microdesc_cache();
951  memcpy(search.digest, d, DIGEST256_LEN);
952  md = HT_FIND(microdesc_map, &cache->map, &search);
953  return md;
954 }
955 
956 /** Return a smartlist of all the sha256 digest of the microdescriptors that
957  * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
958  * to internals of <b>ns</b>; you should not free the members of the resulting
959  * smartlist. Omit all microdescriptors whose digest appear in <b>skip</b>. */
960 smartlist_t *
962  int downloadable_only, digest256map_t *skip)
963 {
964  smartlist_t *result = smartlist_new();
965  time_t now = time(NULL);
966  tor_assert(ns->flavor == FLAV_MICRODESC);
968  if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
969  continue;
970  if (downloadable_only &&
971  !download_status_is_ready(&rs->dl_status, now))
972  continue;
973  if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
974  continue;
975  if (fast_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
976  continue;
977  /* XXXX Also skip if we're a noncache and wouldn't use this router.
978  * XXXX NM Microdesc
979  */
980  smartlist_add(result, rs->descriptor_digest);
981  } SMARTLIST_FOREACH_END(rs);
982  return result;
983 }
984 
985 /** Launch download requests for microdescriptors as appropriate.
986  *
987  * Specifically, we should launch download requests if we are configured to
988  * download mirodescriptors, and there are some microdescriptors listed in the
989  * current microdesc consensus that we don't have, and either we never asked
990  * for them, or we failed to download them but we're willing to retry.
991  */
992 void
994 {
995  const or_options_t *options = get_options();
996  networkstatus_t *consensus;
997  smartlist_t *missing;
998  digest256map_t *pending;
999 
1000  if (should_delay_dir_fetches(options, NULL))
1001  return;
1002  if (dirclient_too_idle_to_fetch_descriptors(options, now))
1003  return;
1004 
1005  /* Give up if we don't have a reasonably live consensus. */
1006  consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
1007  if (!consensus)
1008  return;
1009 
1010  if (!we_fetch_microdescriptors(options))
1011  return;
1012 
1013  pending = digest256map_new();
1015 
1016  missing = microdesc_list_missing_digest256(consensus,
1018  1,
1019  pending);
1020  digest256map_free(pending, NULL);
1021 
1023  missing, NULL, now);
1024 
1025  smartlist_free(missing);
1026 }
1027 
1028 /** For every microdescriptor listed in the current microdescriptor consensus,
1029  * update its last_listed field to be at least as recent as the publication
1030  * time of the current microdescriptor consensus.
1031  */
1032 void
1034 {
1036  microdesc_t *md;
1037  networkstatus_t *ns =
1038  networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
1039 
1040  if (! ns)
1041  return;
1042 
1043  tor_assert(ns->flavor == FLAV_MICRODESC);
1044 
1046  md = microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest);
1047  if (md && ns->valid_after > md->last_listed)
1048  md->last_listed = ns->valid_after;
1049  } SMARTLIST_FOREACH_END(rs);
1050 }
1051 
1052 /** Return true iff we should prefer to use microdescriptors rather than
1053  * routerdescs for building circuits. */
1054 int
1056 {
1057  if (options->UseMicrodescriptors == 0)
1058  return 0; /* the user explicitly picked no */
1059  return 1; /* yes and auto both mean yes */
1060 }
1061 
1062 /** Return true iff we should try to download microdescriptors at all. */
1063 int
1065 {
1066  if (directory_caches_dir_info(options))
1067  return 1;
1068  if (options->FetchUselessDescriptors)
1069  return 1;
1070  return we_use_microdescriptors_for_circuits(options);
1071 }
1072 
1073 /** Return true iff we should try to download router descriptors at all. */
1074 int
1076 {
1077  if (directory_caches_dir_info(options))
1078  return 1;
1079  if (options->FetchUselessDescriptors)
1080  return 1;
1081  return ! we_use_microdescriptors_for_circuits(options);
1082 }
1083 
1084 /** Return the consensus flavor we actually want to use to build circuits. */
1085 MOCK_IMPL(int,
1087 {
1089  return FLAV_MICRODESC;
1090  } else {
1091  return FLAV_NS;
1092  }
1093 }
time_t approx_time(void)
Definition: approx_time.c:32
const char * hex_str(const char *from, size_t fromlen)
Definition: binascii.c:34
void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:478
Header file for circuitbuild.c.
const or_options_t * get_options(void)
Definition: config.c:919
Header file for config.c.
#define HEX_DIGEST_LEN
Definition: crypto_digest.h:35
int tor_memeq(const void *a, const void *b, size_t sz)
Definition: di_ops.c:107
#define fast_memeq(a, b, c)
Definition: di_ops.h:35
#define fast_memneq(a, b, c)
Definition: di_ops.h:42
#define DIGEST_LEN
Definition: digest_sizes.h:20
#define DIGEST256_LEN
Definition: digest_sizes.h:23
int dirclient_too_idle_to_fetch_descriptors(const or_options_t *options, time_t now)
Header for feature/dirclient/dirclient_modes.c.
Header file for directory.c.
#define DIR_PURPOSE_FETCH_MICRODESC
Definition: directory.h:65
dir_server_t * router_get_trusteddirserver_by_digest(const char *digest)
Definition: dirlist.c:160
Header file for dirlist.c.
int directory_caches_dir_info(const or_options_t *options)
Definition: dirserv.c:94
Header file for dirserv.c.
int download_status_is_ready(download_status_t *dls, time_t now)
Definition: dlstatus.c:381
void download_status_mark_impossible(download_status_t *dl)
Definition: dlstatus.c:393
Header file for dlstatus.c.
Header file for circuitbuild.c.
const char * escaped(const char *s)
Definition: escape.c:126
off_t tor_fd_getpos(int fd)
Definition: fdio.c:48
int tor_fd_setpos(int fd, off_t pos)
Definition: fdio.c:80
Header for fdio.c.
int write_str_to_file(const char *fname, const char *str, int bin)
Definition: files.c:274
#define RFTS_IGNORE_MISSING
Definition: files.h:101
int finish_writing_to_file(open_file_t *file_data)
Definition: files.c:465
int start_writing_to_file(const char *fname, int open_flags, int mode, open_file_t **data_out)
Definition: files.c:317
ssize_t write_all_to_fd(int fd, const char *buf, size_t count)
Definition: files.c:162
int abort_writing_to_file(open_file_t *file_data)
Definition: files.c:473
HT_PROTOTYPE(hs_circuitmap_ht, circuit_t, hs_circuitmap_node, hs_circuit_hash_token, hs_circuits_have_same_token)
typedef HT_HEAD(hs_service_ht, hs_service_t) hs_service_ht
#define log_fn(severity, domain, args,...)
Definition: log.h:283
#define LD_FS
Definition: log.h:70
#define LD_BUG
Definition: log.h:86
#define LD_GENERAL
Definition: log.h:62
#define LD_DIR
Definition: log.h:88
void tor_free_(void *mem)
Definition: malloc.c:227
void * tor_reallocarray_(void *ptr, size_t sz1, size_t sz2)
Definition: malloc.c:146
#define tor_free(p)
Definition: malloc.h:52
static int microdesc_eq_(microdesc_t *a, microdesc_t *b)
Definition: microdesc.c:87
void microdesc_free_(microdesc_t *md, const char *fname, int lineno)
Definition: microdesc.c:866
void microdesc_reset_outdated_dirservers_list(void)
Definition: microdesc.c:186
void microdesc_free_all(void)
Definition: microdesc.c:928
int usable_consensus_flavor(void)
Definition: microdesc.c:1086
static unsigned int microdesc_hash_(microdesc_t *md)
Definition: microdesc.c:79
int we_fetch_router_descriptors(const or_options_t *options)
Definition: microdesc.c:1075
void microdesc_note_outdated_dirserver(const char *relay_digest)
Definition: microdesc.c:111
smartlist_t * microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, int no_save)
Definition: microdesc.c:383
void microdesc_check_counts(void)
Definition: microdesc.c:844
static microdesc_cache_t * get_microdesc_cache_noload(void)
Definition: microdesc.c:263
int microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
Definition: microdesc.c:705
static smartlist_t * outdated_dirserver_list
Definition: microdesc.c:106
void microdesc_cache_clear(microdesc_cache_t *cache)
Definition: microdesc.c:465
smartlist_t * microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, int downloadable_only, digest256map_t *skip)
Definition: microdesc.c:961
static ssize_t dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
Definition: microdesc.c:203
int microdesc_cache_reload(microdesc_cache_t *cache)
Definition: microdesc.c:515
microdesc_t * microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
Definition: microdesc.c:946
smartlist_t * microdescs_add_to_cache(microdesc_cache_t *cache, const char *s, const char *eos, saved_location_t where, int no_save, time_t listed_at, smartlist_t *requested_digests256)
Definition: microdesc.c:293
microdesc_cache_t * get_microdesc_cache(void)
Definition: microdesc.c:251
static void microdesc_wipe_body(microdesc_t *md)
Definition: microdesc.c:685
#define TOLERATE_MICRODESC_AGE
Definition: microdesc.c:563
void update_microdesc_downloads(time_t now)
Definition: microdesc.c:993
int microdesc_relay_is_outdated_dirserver(const char *relay_digest)
Definition: microdesc.c:163
void update_microdescs_from_networkstatus(time_t now)
Definition: microdesc.c:1033
static microdesc_cache_t * the_microdesc_cache
Definition: microdesc.c:247
int we_fetch_microdescriptors(const or_options_t *options)
Definition: microdesc.c:1064
int we_use_microdescriptors_for_circuits(const or_options_t *options)
Definition: microdesc.c:1055
void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
Definition: microdesc.c:572
Header file for microdesc.c.
smartlist_t * microdescs_parse_from_string(const char *s, const char *eos, int allow_annotations, saved_location_t where, smartlist_t *invalid_digests_out)
Header file for microdesc_parse.c.
Microdescriptor structure.
networkstatus_t * networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
networkstatus_t * networkstatus_get_reasonably_live_consensus(time_t now, int flavor)
networkstatus_t * networkstatus_get_latest_consensus(void)
int should_delay_dir_fetches(const or_options_t *options, const char **msg_out)
routerstatus_t * router_get_mutable_consensus_status_by_descriptor_digest(networkstatus_t *consensus, const char *digest)
Header file for networkstatus.c.
Networkstatus consensus/vote structure.
Node information structure.
Header file for nodefamily.c.
void router_dir_info_changed(void)
Definition: nodelist.c:2470
const smartlist_t * nodelist_get_list(void)
Definition: nodelist.c:1047
smartlist_t * nodelist_find_nodes_with_microdesc(const microdesc_t *md)
Definition: nodelist.c:863
node_t * nodelist_add_microdesc(microdesc_t *md)
Definition: nodelist.c:635
Header file for nodelist.c.
Master header file for Tor-specific functionality.
saved_location_t
Definition: or.h:627
@ SAVED_IN_JOURNAL
Definition: or.h:641
@ SAVED_NOWHERE
Definition: or.h:630
@ SAVED_IN_CACHE
Definition: or.h:634
Header file for policies.c.
int tor_snprintf(char *str, size_t size, const char *format,...)
Definition: printf.c:27
Header file for router.c.
void list_pending_microdesc_downloads(digest256map_t *result)
Definition: routerlist.c:2396
void launch_descriptor_downloads(int purpose, smartlist_t *downloadable, const routerstatus_t *source, time_t now)
Definition: routerlist.c:2520
Header file for routerlist.c.
Routerstatus (consensus entry) structure.
int smartlist_contains_string(const smartlist_t *sl, const char *element)
Definition: smartlist.c:93
smartlist_t * smartlist_new(void)
void smartlist_add_strdup(struct smartlist_t *sl, const char *string)
void smartlist_add(smartlist_t *sl, void *element)
void smartlist_clear(smartlist_t *sl)
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
#define SMARTLIST_FOREACH(sl, type, var, cmd)
#define SMARTLIST_DEL_CURRENT(sl, var)
unsigned int held_by_nodes
Definition: microdesc_st.h:46
char * onion_pkey
Definition: microdesc_st.h:70
char * body
Definition: microdesc_st.h:58
struct short_policy_t * exit_policy
Definition: microdesc_st.h:85
unsigned int held_in_map
Definition: microdesc_st.h:42
saved_location_bitfield_t saved_location
Definition: microdesc_st.h:38
unsigned int no_save
Definition: microdesc_st.h:40
time_t last_listed
Definition: microdesc_st.h:36
struct curve25519_public_key_t * onion_curve25519_pkey
Definition: microdesc_st.h:75
char digest[DIGEST256_LEN]
Definition: microdesc_st.h:62
size_t bodylen
Definition: microdesc_st.h:60
struct nodefamily_t * family
Definition: microdesc_st.h:83
struct ed25519_public_key_t * ed25519_identity_pkey
Definition: microdesc_st.h:77
struct short_policy_t * ipv6_exit_policy
Definition: microdesc_st.h:87
smartlist_t * routerstatus_list
consensus_flavor_t flavor
Definition: node_st.h:34
int FetchUselessDescriptors
int UseMicrodescriptors
char descriptor_digest[DIGEST256_LEN]
size_t size
Definition: mmap.h:27
const char * data
Definition: mmap.h:26
#define MOCK_IMPL(rv, funcname, arglist)
Definition: testsupport.h:133
void format_iso_time(char *buf, time_t t)
Definition: time_fmt.c:295
#define tor_assert(expr)
Definition: util_bug.h:102
#define tor_fragile_assert()
Definition: util_bug.h:270
int fast_mem_is_zero(const char *mem, size_t len)
Definition: util_string.c:74