tor  0.4.0.0-alpha-dev
microdesc.c
Go to the documentation of this file.
1 /* Copyright (c) 2009-2018, The Tor Project, Inc. */
2 /* See LICENSE for licensing information */
3 
11 #include "core/or/or.h"
12 
13 #include "lib/fdio/fdio.h"
14 
15 #include "app/config/config.h"
16 #include "core/or/circuitbuild.h"
17 #include "core/or/policies.h"
29 #include "feature/relay/router.h"
30 
31 #include "feature/nodelist/microdesc_st.h"
32 #include "feature/nodelist/networkstatus_st.h"
33 #include "feature/nodelist/node_st.h"
34 #include "feature/nodelist/routerstatus_st.h"
35 
36 #ifdef HAVE_FCNTL_H
37 #include <fcntl.h>
38 #endif
39 #ifdef HAVE_SYS_STAT_H
40 #include <sys/stat.h>
41 #endif
42 
50  HT_HEAD(microdesc_map, microdesc_t) map;
51 
53  char *cache_fname;
55  char *journal_fname;
57  tor_mmap_t *cache_content;
59  size_t journal_len;
61  size_t bytes_dropped;
62 
64  uint64_t total_len_seen;
66  unsigned n_seen;
67 
69  int is_loaded;
70 };
71 
73 
75 static inline unsigned int
77 {
78  return (unsigned) siphash24g(md->digest, sizeof(md->digest));
79 }
80 
83 static inline int
85 {
86  return tor_memeq(a->digest, b->digest, DIGEST256_LEN);
87 }
88 
89 HT_PROTOTYPE(microdesc_map, microdesc_t, node,
91 HT_GENERATE2(microdesc_map, microdesc_t, node,
94 
95 /************************* md fetch fail cache *****************************/
96 
97 /* If we end up with too many outdated dirservers, something probably went
98  * wrong so clean up the list. */
99 #define TOO_MANY_OUTDATED_DIRSERVERS 30
100 
103 static smartlist_t *outdated_dirserver_list = NULL;
104 
107 void
108 microdesc_note_outdated_dirserver(const char *relay_digest)
109 {
110  char relay_hexdigest[HEX_DIGEST_LEN+1];
111 
112  /* If we have a reasonably live consensus, then most of our dirservers should
113  * still be caching all the microdescriptors in it. Reasonably live
114  * consensuses are up to a day old (or a day in the future). But
115  * microdescriptors expire 7 days after the last consensus that referenced
116  * them. */
117  if (!networkstatus_get_reasonably_live_consensus(approx_time(),
118  FLAV_MICRODESC)) {
119  return;
120  }
121 
122  if (!outdated_dirserver_list) {
123  outdated_dirserver_list = smartlist_new();
124  }
125 
126  tor_assert(outdated_dirserver_list);
127 
128  /* If the list grows too big, clean it up */
129  if (BUG(smartlist_len(outdated_dirserver_list) >
130  TOO_MANY_OUTDATED_DIRSERVERS)) {
132  }
133 
134  /* Turn the binary relay digest to a hex since smartlists have better support
135  * for strings than digests. */
136  base16_encode(relay_hexdigest,sizeof(relay_hexdigest),
137  relay_digest, DIGEST_LEN);
138 
139  /* Make sure we don't add a dirauth as an outdated dirserver */
140  if (router_get_trusteddirserver_by_digest(relay_digest)) {
141  log_info(LD_GENERAL, "Auth %s gave us outdated dirinfo.", relay_hexdigest);
142  return;
143  }
144 
145  /* Don't double-add outdated dirservers */
146  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
147  return;
148  }
149 
150  /* Add it to the list of outdated dirservers */
151  smartlist_add_strdup(outdated_dirserver_list, relay_hexdigest);
152 
153  log_info(LD_GENERAL, "Noted %s as outdated md dirserver", relay_hexdigest);
154 }
155 
158 int
159 microdesc_relay_is_outdated_dirserver(const char *relay_digest)
160 {
161  char relay_hexdigest[HEX_DIGEST_LEN+1];
162 
163  if (!outdated_dirserver_list) {
164  return 0;
165  }
166 
167  /* Convert identity digest to hex digest */
168  base16_encode(relay_hexdigest, sizeof(relay_hexdigest),
169  relay_digest, DIGEST_LEN);
170 
171  /* Last time we tried to fetch microdescs, was this directory mirror missing
172  * any mds we asked for? */
173  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
174  return 1;
175  }
176 
177  return 0;
178 }
179 
181 void
183 {
184  if (!outdated_dirserver_list) {
185  return;
186  }
187 
188  SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
189  smartlist_clear(outdated_dirserver_list);
190 }
191 
192 /****************************************************************************/
193 
198 static ssize_t
199 dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
200 {
201  ssize_t r = 0;
202  ssize_t written;
203  if (md->body == NULL) {
204  *annotation_len_out = 0;
205  return 0;
206  }
207  /* XXXX drops unknown annotations. */
208  if (md->last_listed) {
209  char buf[ISO_TIME_LEN+1];
210  char annotation[ISO_TIME_LEN+32];
211  format_iso_time(buf, md->last_listed);
212  tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
213  if (write_all_to_fd(fd, annotation, strlen(annotation)) < 0) {
214  log_warn(LD_DIR,
215  "Couldn't write microdescriptor annotation: %s",
216  strerror(errno));
217  return -1;
218  }
219  r += strlen(annotation);
220  *annotation_len_out = r;
221  } else {
222  *annotation_len_out = 0;
223  }
224 
225  md->off = tor_fd_getpos(fd);
226  written = write_all_to_fd(fd, md->body, md->bodylen);
227  if (written != (ssize_t)md->bodylen) {
228  written = written < 0 ? 0 : written;
229  log_warn(LD_DIR,
230  "Couldn't dump microdescriptor (wrote %ld out of %lu): %s",
231  (long)written, (unsigned long)md->bodylen,
232  strerror(errno));
233  return -1;
234  }
235  r += md->bodylen;
236  return r;
237 }
238 
242 
246 {
248  if (PREDICT_UNLIKELY(cache->is_loaded == 0)) {
249  microdesc_cache_reload(cache);
250  }
251  return cache;
252 }
253 
256 static microdesc_cache_t *
258 {
259  if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
260  microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache));
261  HT_INIT(microdesc_map, &cache->map);
262  cache->cache_fname = get_cachedir_fname("cached-microdescs");
263  cache->journal_fname = get_cachedir_fname("cached-microdescs.new");
264  the_microdesc_cache = cache;
265  }
266  return the_microdesc_cache;
267 }
268 
269 /* There are three sources of microdescriptors:
270  1) Generated by us while acting as a directory authority.
271  2) Loaded from the cache on disk.
272  3) Downloaded.
273 */
274 
286 smartlist_t *
288  const char *s, const char *eos, saved_location_t where,
289  int no_save, time_t listed_at,
290  smartlist_t *requested_digests256)
291 {
292  void * const DIGEST_REQUESTED = (void*)1;
293  void * const DIGEST_RECEIVED = (void*)2;
294  void * const DIGEST_INVALID = (void*)3;
295 
296  smartlist_t *descriptors, *added;
297  const int allow_annotations = (where != SAVED_NOWHERE);
298  smartlist_t *invalid_digests = smartlist_new();
299 
300  descriptors = microdescs_parse_from_string(s, eos,
301  allow_annotations,
302  where, invalid_digests);
303  if (listed_at != (time_t)-1) {
304  SMARTLIST_FOREACH(descriptors, microdesc_t *, md,
305  md->last_listed = listed_at);
306  }
307  if (requested_digests256) {
308  digest256map_t *requested;
309  requested = digest256map_new();
310  /* Set requested[d] to DIGEST_REQUESTED for every md we requested. */
311  SMARTLIST_FOREACH(requested_digests256, const uint8_t *, cp,
312  digest256map_set(requested, cp, DIGEST_REQUESTED));
313  /* Set requested[d] to DIGEST_INVALID for every md we requested which we
314  * will never be able to parse. Remove the ones we didn't request from
315  * invalid_digests.
316  */
317  SMARTLIST_FOREACH_BEGIN(invalid_digests, uint8_t *, cp) {
318  if (digest256map_get(requested, cp)) {
319  digest256map_set(requested, cp, DIGEST_INVALID);
320  } else {
321  tor_free(cp);
322  SMARTLIST_DEL_CURRENT(invalid_digests, cp);
323  }
324  } SMARTLIST_FOREACH_END(cp);
325  /* Update requested[d] to 2 for the mds we asked for and got. Delete the
326  * ones we never requested from the 'descriptors' smartlist.
327  */
328  SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
329  if (digest256map_get(requested, (const uint8_t*)md->digest)) {
330  digest256map_set(requested, (const uint8_t*)md->digest,
331  DIGEST_RECEIVED);
332  } else {
333  log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microdesc");
334  microdesc_free(md);
335  SMARTLIST_DEL_CURRENT(descriptors, md);
336  }
337  } SMARTLIST_FOREACH_END(md);
338  /* Remove the ones we got or the invalid ones from requested_digests256.
339  */
340  SMARTLIST_FOREACH_BEGIN(requested_digests256, uint8_t *, cp) {
341  void *status = digest256map_get(requested, cp);
342  if (status == DIGEST_RECEIVED || status == DIGEST_INVALID) {
343  tor_free(cp);
344  SMARTLIST_DEL_CURRENT(requested_digests256, cp);
345  }
346  } SMARTLIST_FOREACH_END(cp);
347  digest256map_free(requested, NULL);
348  }
349 
350  /* For every requested microdescriptor that was unparseable, mark it
351  * as not to be retried. */
352  if (smartlist_len(invalid_digests)) {
353  networkstatus_t *ns =
354  networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
355  if (ns) {
356  SMARTLIST_FOREACH_BEGIN(invalid_digests, char *, d) {
357  routerstatus_t *rs =
358  router_get_mutable_consensus_status_by_descriptor_digest(ns, d);
359  if (rs && tor_memeq(d, rs->descriptor_digest, DIGEST256_LEN)) {
360  download_status_mark_impossible(&rs->dl_status);
361  }
362  } SMARTLIST_FOREACH_END(d);
363  }
364  }
365  SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
366  smartlist_free(invalid_digests);
367 
368  added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
369  smartlist_free(descriptors);
370  return added;
371 }
372 
376 smartlist_t *
378  smartlist_t *descriptors, saved_location_t where,
379  int no_save)
380 {
381  smartlist_t *added;
382  open_file_t *open_file = NULL;
383  int fd = -1;
384  // int n_added = 0;
385  ssize_t size = 0;
386 
387  if (where == SAVED_NOWHERE && !no_save) {
388  fd = start_writing_to_file(cache->journal_fname,
389  OPEN_FLAGS_APPEND|O_BINARY,
390  0600, &open_file);
391  if (fd < 0) {
392  log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
393  cache->journal_fname, strerror(errno));
394  }
395  }
396 
397  added = smartlist_new();
398  SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
399  microdesc_t *md2;
400  md2 = HT_FIND(microdesc_map, &cache->map, md);
401  if (md2) {
402  /* We already had this one. */
403  if (md2->last_listed < md->last_listed)
404  md2->last_listed = md->last_listed;
405  microdesc_free(md);
406  if (where != SAVED_NOWHERE)
407  cache->bytes_dropped += size;
408  continue;
409  }
410 
411  /* Okay, it's a new one. */
412  if (fd >= 0) {
413  size_t annotation_len;
414  size = dump_microdescriptor(fd, md, &annotation_len);
415  if (size < 0) {
416  /* we already warned in dump_microdescriptor */
417  abort_writing_to_file(open_file);
418  fd = -1;
419  } else {
420  md->saved_location = SAVED_IN_JOURNAL;
421  cache->journal_len += size;
422  }
423  } else {
424  md->saved_location = where;
425  }
426 
427  md->no_save = no_save;
428 
429  HT_INSERT(microdesc_map, &cache->map, md);
430  md->held_in_map = 1;
431  smartlist_add(added, md);
432  ++cache->n_seen;
433  cache->total_len_seen += md->bodylen;
434  } SMARTLIST_FOREACH_END(md);
435 
436  if (fd >= 0) {
437  if (finish_writing_to_file(open_file) < 0) {
438  log_warn(LD_DIR, "Error appending to microdescriptor file: %s",
439  strerror(errno));
440  smartlist_clear(added);
441  return added;
442  }
443  }
444 
445  {
446  networkstatus_t *ns = networkstatus_get_latest_consensus();
447  if (ns && ns->flavor == FLAV_MICRODESC)
449  }
450 
451  if (smartlist_len(added))
453 
454  return added;
455 }
456 
458 void
460 {
461  microdesc_t **entry, **next;
462 
463  for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
464  microdesc_t *md = *entry;
465  next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
466  md->held_in_map = 0;
467  microdesc_free(md);
468  }
469  HT_CLEAR(microdesc_map, &cache->map);
470  if (cache->cache_content) {
471  int res = tor_munmap_file(cache->cache_content);
472  if (res != 0) {
473  log_warn(LD_FS,
474  "tor_munmap_file() failed clearing microdesc cache; "
475  "we are probably about to leak memory.");
476  /* TODO something smarter? */
477  }
478  cache->cache_content = NULL;
479  }
480  cache->total_len_seen = 0;
481  cache->n_seen = 0;
482  cache->bytes_dropped = 0;
483 }
484 
487 int
489 {
490  struct stat st;
491  char *journal_content;
492  smartlist_t *added;
493  tor_mmap_t *mm;
494  int total = 0;
495 
496  microdesc_cache_clear(cache);
497 
498  cache->is_loaded = 1;
499 
500  mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
501  if (mm) {
502  added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
503  SAVED_IN_CACHE, 0, -1, NULL);
504  if (added) {
505  total += smartlist_len(added);
506  smartlist_free(added);
507  }
508  }
509 
510  journal_content = read_file_to_str(cache->journal_fname,
511  RFTS_IGNORE_MISSING, &st);
512  if (journal_content) {
513  cache->journal_len = (size_t) st.st_size;
514  added = microdescs_add_to_cache(cache, journal_content,
515  journal_content+st.st_size,
516  SAVED_IN_JOURNAL, 0, -1, NULL);
517  if (added) {
518  total += smartlist_len(added);
519  smartlist_free(added);
520  }
521  tor_free(journal_content);
522  }
523  log_info(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
524  total);
525 
526  microdesc_cache_rebuild(cache, 0 /* don't force */);
527 
528  return 0;
529 }
530 
533 #define TOLERATE_MICRODESC_AGE (7*24*60*60)
534 
541 void
542 microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
543 {
544  microdesc_t **mdp, *victim;
545  int dropped=0, kept=0;
546  size_t bytes_dropped = 0;
547  time_t now = time(NULL);
548 
549  /* If we don't know a reasonably live consensus, don't believe last_listed
550  * values: we might be starting up after being down for a while. */
551  if (! force &&
552  ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
553  return;
554 
555  if (cutoff <= 0)
556  cutoff = now - TOLERATE_MICRODESC_AGE;
557 
558  for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
559  const int is_old = (*mdp)->last_listed < cutoff;
560  const unsigned held_by_nodes = (*mdp)->held_by_nodes;
561  if (is_old && !held_by_nodes) {
562  ++dropped;
563  victim = *mdp;
564  mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
565  victim->held_in_map = 0;
566  bytes_dropped += victim->bodylen;
567  microdesc_free(victim);
568  } else {
569  if (is_old) {
570  /* It's old, but it has held_by_nodes set. That's not okay. */
571  /* Let's try to diagnose and fix #7164 . */
573  const networkstatus_t *ns = networkstatus_get_latest_consensus();
574  long networkstatus_age = -1;
575  const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
576  if (ns) {
577  networkstatus_age = now - ns->valid_after;
578  }
579  log_warn(LD_BUG, "Microdescriptor seemed very old "
580  "(last listed %d hours ago vs %d hour cutoff), but is still "
581  "marked as being held by %d node(s). I found %d node(s) "
582  "holding it. Current networkstatus is %ld hours old. "
583  "Hashtable badness is %d.",
584  (int)((now - (*mdp)->last_listed) / 3600),
585  (int)((now - cutoff) / 3600),
586  held_by_nodes,
587  smartlist_len(nodes),
588  networkstatus_age / 3600,
589  ht_badness);
590 
591  SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
592  const char *rs_match = "No RS";
593  const char *rs_present = "";
594  if (node->rs) {
595  if (tor_memeq(node->rs->descriptor_digest,
596  (*mdp)->digest, DIGEST256_LEN)) {
597  rs_match = "Microdesc digest in RS matches";
598  } else {
599  rs_match = "Microdesc digest in RS does match";
600  }
601  if (ns) {
602  /* This should be impossible, but let's see! */
603  rs_present = " RS not present in networkstatus.";
605  if (rs == node->rs) {
606  rs_present = " RS okay in networkstatus.";
607  }
608  });
609  }
610  }
611  log_warn(LD_BUG, " [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s",
612  node_sl_idx,
613  hex_str(node->identity, DIGEST_LEN),
614  node->md, node->rs, node->ri, rs_match, rs_present);
615  } SMARTLIST_FOREACH_END(node);
616  smartlist_free(nodes);
617  (*mdp)->last_listed = now;
618  }
619 
620  ++kept;
621  mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
622  }
623  }
624 
625  if (dropped) {
626  log_info(LD_DIR, "Removed %d/%d microdescriptors as old.",
627  dropped,dropped+kept);
628  cache->bytes_dropped += bytes_dropped;
629  }
630 }
631 
632 static int
633 should_rebuild_md_cache(microdesc_cache_t *cache)
634 {
635  const size_t old_len =
636  cache->cache_content ? cache->cache_content->size : 0;
637  const size_t journal_len = cache->journal_len;
638  const size_t dropped = cache->bytes_dropped;
639 
640  if (journal_len < 16384)
641  return 0; /* Don't bother, not enough has happened yet. */
642  if (dropped > (journal_len + old_len) / 3)
643  return 1; /* We could save 1/3 or more of the currently used space. */
644  if (journal_len > old_len / 2)
645  return 1; /* We should append to the regular file */
646 
647  return 0;
648 }
649 
654 static void
656 {
657  if (!md)
658  return;
659 
660  if (md->saved_location != SAVED_IN_CACHE)
661  tor_free(md->body);
662 
663  md->off = 0;
665  md->body = NULL;
666  md->bodylen = 0;
667  md->no_save = 1;
668 }
669 
674 int
676 {
677  open_file_t *open_file;
678  int fd = -1, res;
679  microdesc_t **mdp;
680  smartlist_t *wrote;
681  ssize_t size;
682  off_t off = 0, off_real;
683  int orig_size, new_size;
684 
685  if (cache == NULL) {
686  cache = the_microdesc_cache;
687  if (cache == NULL)
688  return 0;
689  }
690 
691  /* Remove dead descriptors */
692  microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);
693 
694  if (!force && !should_rebuild_md_cache(cache))
695  return 0;
696 
697  log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
698 
699  orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
700  orig_size += (int)cache->journal_len;
701 
702  fd = start_writing_to_file(cache->cache_fname,
703  OPEN_FLAGS_REPLACE|O_BINARY,
704  0600, &open_file);
705  if (fd < 0)
706  return -1;
707 
708  wrote = smartlist_new();
709 
710  HT_FOREACH(mdp, microdesc_map, &cache->map) {
711  microdesc_t *md = *mdp;
712  size_t annotation_len;
713  if (md->no_save || !md->body)
714  continue;
715 
716  size = dump_microdescriptor(fd, md, &annotation_len);
717  if (size < 0) {
719 
720  /* rewind, in case it was a partial write. */
721  tor_fd_setpos(fd, off);
722  continue;
723  }
724  tor_assert(((size_t)size) == annotation_len + md->bodylen);
725  md->off = off + annotation_len;
726  off += size;
727  off_real = tor_fd_getpos(fd);
728  if (off_real != off) {
729  log_warn(LD_BUG, "Discontinuity in position in microdescriptor cache."
730  "By my count, I'm at %"PRId64
731  ", but I should be at %"PRId64,
732  (int64_t)(off), (int64_t)(off_real));
733  if (off_real >= 0)
734  off = off_real;
735  }
736  if (md->saved_location != SAVED_IN_CACHE) {
737  tor_free(md->body);
739  }
740  smartlist_add(wrote, md);
741  }
742 
743  /* We must do this unmap _before_ we call finish_writing_to_file(), or
744  * windows will not actually replace the file. */
745  if (cache->cache_content) {
746  res = tor_munmap_file(cache->cache_content);
747  if (res != 0) {
748  log_warn(LD_FS,
749  "Failed to unmap old microdescriptor cache while rebuilding");
750  }
751  cache->cache_content = NULL;
752  }
753 
754  if (finish_writing_to_file(open_file) < 0) {
755  log_warn(LD_DIR, "Error rebuilding microdescriptor cache: %s",
756  strerror(errno));
757  /* Okay. Let's prevent from making things worse elsewhere. */
758  cache->cache_content = NULL;
759  HT_FOREACH(mdp, microdesc_map, &cache->map) {
760  microdesc_t *md = *mdp;
761  if (md->saved_location == SAVED_IN_CACHE) {
763  }
764  }
765  smartlist_free(wrote);
766  return -1;
767  }
768 
769  cache->cache_content = tor_mmap_file(cache->cache_fname);
770 
771  if (!cache->cache_content && smartlist_len(wrote)) {
772  log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
773  cache->cache_fname);
774  smartlist_free(wrote);
775  return -1;
776  }
777  SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
778  tor_assert(md->saved_location == SAVED_IN_CACHE);
779  md->body = (char*)cache->cache_content->data + md->off;
780  if (PREDICT_UNLIKELY(
781  md->bodylen < 9 || fast_memneq(md->body, "onion-key", 9) != 0)) {
782  /* XXXX once bug 2022 is solved, we can kill this block and turn it
783  * into just the tor_assert(fast_memeq) */
784  off_t avail = cache->cache_content->size - md->off;
785  char *bad_str;
786  tor_assert(avail >= 0);
787  bad_str = tor_strndup(md->body, MIN(128, (size_t)avail));
788  log_err(LD_BUG, "After rebuilding microdesc cache, offsets seem wrong. "
789  " At offset %d, I expected to find a microdescriptor starting "
790  " with \"onion-key\". Instead I got %s.",
791  (int)md->off, escaped(bad_str));
792  tor_free(bad_str);
793  tor_assert(fast_memeq(md->body, "onion-key", 9));
794  }
795  } SMARTLIST_FOREACH_END(md);
796 
797  smartlist_free(wrote);
798 
799  write_str_to_file(cache->journal_fname, "", 1);
800  cache->journal_len = 0;
801  cache->bytes_dropped = 0;
802 
803  new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
804  log_info(LD_DIR, "Done rebuilding microdesc cache. "
805  "Saved %d bytes; %d still used.",
806  orig_size-new_size, new_size);
807 
808  return 0;
809 }
810 
813 void
815 {
816  microdesc_t **mdp;
817  if (!the_microdesc_cache)
818  return;
819 
820  HT_FOREACH(mdp, microdesc_map, &the_microdesc_cache->map) {
821  microdesc_t *md = *mdp;
822  unsigned int found=0;
823  const smartlist_t *nodes = nodelist_get_list();
824  SMARTLIST_FOREACH(nodes, node_t *, node, {
825  if (node->md == md) {
826  ++found;
827  }
828  });
829  tor_assert(found == md->held_by_nodes);
830  }
831 }
832 
835 void
836 microdesc_free_(microdesc_t *md, const char *fname, int lineno)
837 {
838  if (!md)
839  return;
840 
841  /* Make sure that the microdesc was really removed from the appropriate data
842  structures. */
843  if (md->held_in_map) {
845  microdesc_t *md2 = HT_FIND(microdesc_map, &cache->map, md);
846  if (md2 == md) {
847  log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
848  "in microdesc_map", fname, lineno);
849  HT_REMOVE(microdesc_map, &cache->map, md);
850  } else {
851  log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_in_map "
852  "set, but microdesc was not in the map.", fname, lineno);
853  }
855  }
856  if (md->held_by_nodes) {
858  int found=0;
859  const smartlist_t *nodes = nodelist_get_list();
860  const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
861  SMARTLIST_FOREACH(nodes, node_t *, node, {
862  if (node->md == md) {
863  ++found;
864  node->md = NULL;
865  }
866  });
867  if (found) {
868  log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
869  "referenced %d node(s); held_by_nodes == %u, ht_badness == %d",
870  fname, lineno, found, md->held_by_nodes, ht_badness);
871  } else {
872  log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_by_nodes "
873  "set to %u, but md was not referenced by any nodes. "
874  "ht_badness == %d",
875  fname, lineno, md->held_by_nodes, ht_badness);
876  }
878  }
879  //tor_assert(md->held_in_map == 0);
880  //tor_assert(md->held_by_nodes == 0);
881 
882  if (md->onion_pkey)
883  tor_free(md->onion_pkey);
886  if (md->body && md->saved_location != SAVED_IN_CACHE)
887  tor_free(md->body);
888 
889  nodefamily_free(md->family);
890  short_policy_free(md->exit_policy);
891  short_policy_free(md->ipv6_exit_policy);
892 
893  tor_free(md);
894 }
895 
897 void
899 {
900  if (the_microdesc_cache) {
902  tor_free(the_microdesc_cache->cache_fname);
903  tor_free(the_microdesc_cache->journal_fname);
905  }
906 
907  if (outdated_dirserver_list) {
908  SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
909  smartlist_free(outdated_dirserver_list);
910  }
911 }
912 
915 microdesc_t *
917 {
918  microdesc_t *md, search;
919  if (!cache)
920  cache = get_microdesc_cache();
921  memcpy(search.digest, d, DIGEST256_LEN);
922  md = HT_FIND(microdesc_map, &cache->map, &search);
923  return md;
924 }
925 
930 smartlist_t *
932  int downloadable_only, digest256map_t *skip)
933 {
934  smartlist_t *result = smartlist_new();
935  time_t now = time(NULL);
936  tor_assert(ns->flavor == FLAV_MICRODESC);
938  if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
939  continue;
940  if (downloadable_only &&
941  !download_status_is_ready(&rs->dl_status, now))
942  continue;
943  if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
944  continue;
945  if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
946  continue;
947  /* XXXX Also skip if we're a noncache and wouldn't use this router.
948  * XXXX NM Microdesc
949  */
950  smartlist_add(result, rs->descriptor_digest);
951  } SMARTLIST_FOREACH_END(rs);
952  return result;
953 }
954 
962 void
964 {
965  const or_options_t *options = get_options();
966  networkstatus_t *consensus;
967  smartlist_t *missing;
968  digest256map_t *pending;
969 
970  if (should_delay_dir_fetches(options, NULL))
971  return;
972  if (directory_too_idle_to_fetch_descriptors(options, now))
973  return;
974 
975  /* Give up if we don't have a reasonably live consensus. */
976  consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
977  if (!consensus)
978  return;
979 
980  if (!we_fetch_microdescriptors(options))
981  return;
982 
983  pending = digest256map_new();
985 
986  missing = microdesc_list_missing_digest256(consensus,
988  1,
989  pending);
990  digest256map_free(pending, NULL);
991 
993  missing, NULL, now);
994 
995  smartlist_free(missing);
996 }
997 
1002 void
1004 {
1006  microdesc_t *md;
1007  networkstatus_t *ns =
1008  networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
1009 
1010  if (! ns)
1011  return;
1012 
1013  tor_assert(ns->flavor == FLAV_MICRODESC);
1014 
1016  md = microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest);
1017  if (md && ns->valid_after > md->last_listed)
1018  md->last_listed = ns->valid_after;
1019  } SMARTLIST_FOREACH_END(rs);
1020 }
1021 
1024 int
1026 {
1027  if (options->UseMicrodescriptors == 0)
1028  return 0; /* the user explicitly picked no */
1029  return 1; /* yes and auto both mean yes */
1030 }
1031 
1033 int
1035 {
1036  if (directory_caches_dir_info(options))
1037  return 1;
1038  if (options->FetchUselessDescriptors)
1039  return 1;
1040  return we_use_microdescriptors_for_circuits(options);
1041 }
1042 
1044 int
1046 {
1047  if (directory_caches_dir_info(options))
1048  return 1;
1049  if (options->FetchUselessDescriptors)
1050  return 1;
1051  return ! we_use_microdescriptors_for_circuits(options);
1052 }
1053 
1056 usable_consensus_flavor,(void))
1057 {
1058  if (we_use_microdescriptors_for_circuits(get_options())) {
1059  return FLAV_MICRODESC;
1060  } else {
1061  return FLAV_NS;
1062  }
1063 }
Header file for dirserv.c.
void microdesc_free_(microdesc_t *md, const char *fname, int lineno)
Definition: microdesc.c:836
struct short_policy_t * exit_policy
Definition: microdesc_st.h:76
smartlist_t * nodelist_find_nodes_with_microdesc(const microdesc_t *md)
Definition: nodelist.c:767
Header file for circuitbuild.c.
Definition: node_st.h:28
int tor_fd_setpos(int fd, off_t pos)
Definition: fdio.c:75
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
HT_PROTOTYPE(HT_GENERATE2(microdesc_map, HT_GENERATE2(microdesc_t, HT_GENERATE2(node, HT_GENERATE2(microdesc_hash_, HT_GENERATE2(microdesc_eq_)
Definition: microdesc.c:89
Header file for nodefamily.c.
void router_dir_info_changed(void)
Definition: nodelist.c:2234
int we_use_microdescriptors_for_circuits(const or_options_t *options)
Definition: microdesc.c:1025
static ssize_t dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
Definition: microdesc.c:199
void smartlist_add_strdup(struct smartlist_t *sl, const char *string)
int tor_mem_is_zero(const char *mem, size_t len)
Definition: util_string.c:74
void microdesc_check_counts(void)
Definition: microdesc.c:814
#define LD_GENERAL
Definition: log.h:58
void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
Definition: microdesc.c:542
Header file for nodelist.c.
Header for fdio.c.
Header file for directory.c.
void smartlist_add(smartlist_t *sl, void *element)
ssize_t write_all_to_fd(int fd, const char *buf, size_t count)
Definition: files.c:162
void list_pending_microdesc_downloads(digest256map_t *result)
Definition: routerlist.c:2304
Header file for config.c.
smartlist_t * microdescs_add_to_cache(microdesc_cache_t *cache, const char *s, const char *eos, saved_location_t where, int no_save, time_t listed_at, smartlist_t *requested_digests256)
Definition: microdesc.c:287
off_t tor_fd_getpos(int fd)
Definition: fdio.c:43
Header file for microdesc.c.
int we_fetch_router_descriptors(const or_options_t *options)
Definition: microdesc.c:1045
int UseMicrodescriptors
int directory_too_idle_to_fetch_descriptors(const or_options_t *options, time_t now)
Definition: dirserv.c:169
#define tor_free(p)
Definition: malloc.h:52
#define tor_fragile_assert()
Definition: util_bug.h:221
void update_microdescs_from_networkstatus(time_t now)
Definition: microdesc.c:1003
#define SMARTLIST_DEL_CURRENT(sl, var)
size_t size
Definition: mmap.h:26
Header file for microdesc_parse.c.
int download_status_is_ready(download_status_t *dls, time_t now)
Definition: dlstatus.c:376
dir_server_t * router_get_trusteddirserver_by_digest(const char *digest)
Definition: dirlist.c:112
void * tor_reallocarray_(void *ptr, size_t sz1, size_t sz2)
Definition: malloc.c:146
void update_microdesc_downloads(time_t now)
Definition: microdesc.c:963
saved_location_t
Definition: or.h:746
size_t bodylen
Definition: microdesc_st.h:51
void microdesc_reset_outdated_dirservers_list(void)
Definition: microdesc.c:182
time_t last_listed
Definition: microdesc_st.h:29
#define DIGEST256_LEN
Definition: digest_sizes.h:23
Header file for policies.c.
tor_assert(buffer)
node_t * nodelist_add_microdesc(microdesc_t *md)
Definition: nodelist.c:549
smartlist_t * microdescs_parse_from_string(const char *s, const char *eos, int allow_annotations, saved_location_t where, smartlist_t *invalid_digests_out)
static int microdesc_eq_(microdesc_t *a, microdesc_t *b)
Definition: microdesc.c:84
int finish_writing_to_file(open_file_t *file_data)
Definition: files.c:449
int tor_memeq(const void *a, const void *b, size_t sz)
Definition: di_ops.c:107
unsigned int held_by_nodes
Definition: microdesc_st.h:37
void download_status_mark_impossible(download_status_t *dl)
Definition: dlstatus.c:388
#define DIGEST_LEN
Definition: digest_sizes.h:20
Master header file for Tor-specific functionality.
const char * hex_str(const char *from, size_t fromlen)
Definition: binascii.c:34
const char * data
Definition: mmap.h:25
Header file for circuitbuild.c.
#define TOLERATE_MICRODESC_AGE
Definition: microdesc.c:533
void microdesc_cache_clear(microdesc_cache_t *cache)
Definition: microdesc.c:459
int microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
Definition: microdesc.c:675
void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:476
struct ed25519_public_key_t * ed25519_identity_pkey
Definition: microdesc_st.h:68
void tor_free_(void *mem)
Definition: malloc.c:227
char * body
Definition: microdesc_st.h:49
unsigned int held_in_map
Definition: microdesc_st.h:35
#define RFTS_IGNORE_MISSING
Definition: files.h:97
static microdesc_cache_t * the_microdesc_cache
Definition: microdesc.c:241
int start_writing_to_file(const char *fname, int open_flags, int mode, open_file_t **data_out)
Definition: files.c:301
int microdesc_relay_is_outdated_dirserver(const char *relay_digest)
Definition: microdesc.c:159
static void microdesc_wipe_body(microdesc_t *md)
Definition: microdesc.c:655
char * onion_pkey
Definition: microdesc_st.h:61
MOCK_IMPL(int, usable_consensus_flavor,(void))
Definition: microdesc.c:1055
microdesc_t * microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
Definition: microdesc.c:916
int FetchUselessDescriptors
#define LD_FS
Definition: log.h:66
smartlist_t * routerstatus_list
static microdesc_cache_t * get_microdesc_cache_noload(void)
Definition: microdesc.c:257
char descriptor_digest[DIGEST256_LEN]
#define HEX_DIGEST_LEN
Definition: crypto_digest.h:35
#define LD_DIR
Definition: log.h:84
consensus_flavor_t flavor
void launch_descriptor_downloads(int purpose, smartlist_t *downloadable, const routerstatus_t *source, time_t now)
Definition: routerlist.c:2428
int smartlist_contains_string(const smartlist_t *sl, const char *element)
Definition: smartlist.c:93
int tor_snprintf(char *str, size_t size, const char *format,...)
Definition: printf.c:27
void format_iso_time(char *buf, time_t t)
Definition: time_fmt.c:291
microdesc_cache_t * get_microdesc_cache(void)
Definition: microdesc.c:245
struct curve25519_public_key_t * onion_curve25519_pkey
Definition: microdesc_st.h:66
#define SMARTLIST_FOREACH(sl, type, var, cmd)
void microdesc_free_all(void)
Definition: microdesc.c:898
Header file for router.c.
const char * escaped(const char *s)
Definition: escape.c:126
static unsigned int microdesc_hash_(microdesc_t *md)
Definition: microdesc.c:76
smartlist_t * microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, int downloadable_only, digest256map_t *skip)
Definition: microdesc.c:931
struct short_policy_t * ipv6_exit_policy
Definition: microdesc_st.h:78
Header file for dlstatus.c.
#define log_fn(severity, domain, args,...)
Definition: log.h:255
#define DIR_PURPOSE_FETCH_MICRODESC
Definition: directory.h:70
char digest[DIGEST256_LEN]
Definition: microdesc_st.h:53
time_t approx_time(void)
Definition: approx_time.c:32
Header file for dirlist.c.
int abort_writing_to_file(open_file_t *file_data)
Definition: files.c:457
int directory_caches_dir_info(const or_options_t *options)
Definition: dirserv.c:143
int we_fetch_microdescriptors(const or_options_t *options)
Definition: microdesc.c:1034
int microdesc_cache_reload(microdesc_cache_t *cache)
Definition: microdesc.c:488
unsigned int no_save
Definition: microdesc_st.h:33
struct nodefamily_t * family
Definition: microdesc_st.h:74
smartlist_t * microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, int no_save)
Definition: microdesc.c:377
void smartlist_clear(smartlist_t *sl)
saved_location_bitfield_t saved_location
Definition: microdesc_st.h:31
int should_delay_dir_fetches(const or_options_t *options, const char **msg_out)
Header file for networkstatus.c.
#define LD_BUG
Definition: log.h:82
Header file for routerlist.c.