Line data Source code
1 : /* Copyright (c) 2009-2021, The Tor Project, Inc. */
2 : /* See LICENSE for licensing information */
3 :
4 : /**
5 : * \file microdesc.c
6 : *
7 : * \brief Implements microdescriptors -- an abbreviated description of
8 : * less-frequently-changing router information.
9 : */
10 :
11 : #include "core/or/or.h"
12 :
13 : #include "lib/fdio/fdio.h"
14 :
15 : #include "app/config/config.h"
16 : #include "core/or/circuitbuild.h"
17 : #include "core/or/policies.h"
18 : #include "feature/client/entrynodes.h"
19 : #include "feature/dircache/dirserv.h"
20 : #include "feature/dirclient/dlstatus.h"
21 : #include "feature/dirclient/dirclient_modes.h"
22 : #include "feature/dircommon/directory.h"
23 : #include "feature/dirparse/microdesc_parse.h"
24 : #include "feature/nodelist/dirlist.h"
25 : #include "feature/nodelist/microdesc.h"
26 : #include "feature/nodelist/networkstatus.h"
27 : #include "feature/nodelist/nodefamily.h"
28 : #include "feature/nodelist/nodelist.h"
29 : #include "feature/nodelist/routerlist.h"
30 : #include "feature/relay/router.h"
31 :
32 : #include "feature/nodelist/microdesc_st.h"
33 : #include "feature/nodelist/networkstatus_st.h"
34 : #include "feature/nodelist/node_st.h"
35 : #include "feature/nodelist/routerstatus_st.h"
36 :
37 : #ifdef HAVE_FCNTL_H
38 : #include <fcntl.h>
39 : #endif
40 : #ifdef HAVE_SYS_STAT_H
41 : #include <sys/stat.h>
42 : #endif
43 :
44 : /** A data structure to hold a bunch of cached microdescriptors. There are
45 : * two active files in the cache: a "cache file" that we mmap, and a "journal
46 : * file" that we append to. Periodically, we rebuild the cache file to hold
47 : * only the microdescriptors that we want to keep */
48 : struct microdesc_cache_t {
49 : /** Map from sha256-digest to microdesc_t for every microdesc_t in the
50 : * cache. */
51 : HT_HEAD(microdesc_map, microdesc_t) map;
52 :
53 : /** Name of the cache file. */
54 : char *cache_fname;
55 : /** Name of the journal file. */
56 : char *journal_fname;
57 : /** Mmap'd contents of the cache file, or NULL if there is none. */
58 : tor_mmap_t *cache_content;
59 : /** Number of bytes used in the journal file. */
60 : size_t journal_len;
61 : /** Number of bytes in descriptors removed as too old. */
62 : size_t bytes_dropped;
63 :
64 : /** Total bytes of microdescriptor bodies we have added to this cache */
65 : uint64_t total_len_seen;
66 : /** Total number of microdescriptors we have added to this cache */
67 : unsigned n_seen;
68 :
69 : /** True iff we have loaded this cache from disk ever. */
70 : int is_loaded;
71 : };
72 :
73 : static microdesc_cache_t *get_microdesc_cache_noload(void);
74 : static void warn_if_nul_found(const char *inp, size_t len, int64_t offset,
75 : const char *activity);
76 :
77 : /** Helper: computes a hash of <b>md</b> to place it in a hash table. */
78 : static inline unsigned int
79 98 : microdesc_hash_(microdesc_t *md)
80 : {
81 98 : return (unsigned) siphash24g(md->digest, sizeof(md->digest));
82 : }
83 :
84 : /** Helper: compares <b>a</b> and <b>b</b> for equality for hash-table
85 : * purposes. */
86 : static inline int
87 13 : microdesc_eq_(microdesc_t *a, microdesc_t *b)
88 : {
89 13 : return tor_memeq(a->digest, b->digest, DIGEST256_LEN);
90 : }
91 :
92 739 : HT_PROTOTYPE(microdesc_map, microdesc_t, node,
93 : microdesc_hash_, microdesc_eq_);
94 29 : HT_GENERATE2(microdesc_map, microdesc_t, node,
95 : microdesc_hash_, microdesc_eq_, 0.6,
96 : tor_reallocarray_, tor_free_);
97 :
98 : /************************* md fetch fail cache *****************************/
99 :
100 : /* If we end up with too many outdated dirservers, something probably went
101 : * wrong so clean up the list. */
102 : #define TOO_MANY_OUTDATED_DIRSERVERS 30
103 :
104 : /** List of dirservers with outdated microdesc information. The smartlist is
105 : * filled with the hex digests of outdated dirservers. */
106 : static smartlist_t *outdated_dirserver_list = NULL;
107 :
108 : /** Note that we failed to fetch a microdescriptor from the relay with
109 : * <b>relay_digest</b> (of size DIGEST_LEN). */
110 : void
111 9 : microdesc_note_outdated_dirserver(const char *relay_digest)
112 : {
113 9 : char relay_hexdigest[HEX_DIGEST_LEN+1];
114 :
115 : /* If we have a reasonably live consensus, then most of our dirservers should
116 : * still be caching all the microdescriptors in it. Reasonably live
117 : * consensuses are up to a day old (or a day in the future). But
118 : * microdescriptors expire 7 days after the last consensus that referenced
119 : * them. */
120 9 : if (!networkstatus_get_reasonably_live_consensus(approx_time(),
121 : FLAV_MICRODESC)) {
122 0 : return;
123 : }
124 :
125 9 : if (!outdated_dirserver_list) {
126 3 : outdated_dirserver_list = smartlist_new();
127 : }
128 :
129 9 : tor_assert(outdated_dirserver_list);
130 :
131 : /* If the list grows too big, clean it up */
132 9 : if (smartlist_len(outdated_dirserver_list) > TOO_MANY_OUTDATED_DIRSERVERS) {
133 0 : log_info(LD_GENERAL,"Too many outdated directory servers (%d). Resetting.",
134 : smartlist_len(outdated_dirserver_list));
135 0 : microdesc_reset_outdated_dirservers_list();
136 : }
137 :
138 : /* Turn the binary relay digest to a hex since smartlists have better support
139 : * for strings than digests. */
140 9 : base16_encode(relay_hexdigest,sizeof(relay_hexdigest),
141 : relay_digest, DIGEST_LEN);
142 :
143 : /* Make sure we don't add a dirauth as an outdated dirserver */
144 9 : if (router_get_trusteddirserver_by_digest(relay_digest)) {
145 0 : log_info(LD_GENERAL, "Auth %s gave us outdated dirinfo.", relay_hexdigest);
146 0 : return;
147 : }
148 :
149 : /* Don't double-add outdated dirservers */
150 9 : if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
151 : return;
152 : }
153 :
154 : /* Add it to the list of outdated dirservers */
155 9 : smartlist_add_strdup(outdated_dirserver_list, relay_hexdigest);
156 :
157 9 : log_info(LD_GENERAL, "Noted %s as outdated md dirserver", relay_hexdigest);
158 : }
159 :
160 : /** Return True if the relay with <b>relay_digest</b> (size DIGEST_LEN) is an
161 : * outdated dirserver */
162 : int
163 129 : microdesc_relay_is_outdated_dirserver(const char *relay_digest)
164 : {
165 129 : char relay_hexdigest[HEX_DIGEST_LEN+1];
166 :
167 129 : if (!outdated_dirserver_list) {
168 : return 0;
169 : }
170 :
171 : /* Convert identity digest to hex digest */
172 129 : base16_encode(relay_hexdigest, sizeof(relay_hexdigest),
173 : relay_digest, DIGEST_LEN);
174 :
175 : /* Last time we tried to fetch microdescs, was this directory mirror missing
176 : * any mds we asked for? */
177 129 : if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
178 27 : return 1;
179 : }
180 :
181 : return 0;
182 : }
183 :
184 : /** Reset the list of outdated dirservers. */
185 : void
186 5 : microdesc_reset_outdated_dirservers_list(void)
187 : {
188 5 : if (!outdated_dirserver_list) {
189 : return;
190 : }
191 :
192 0 : SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
193 0 : smartlist_clear(outdated_dirserver_list);
194 : }
195 :
196 : /****************************************************************************/
197 :
198 : /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
199 : * On success, return the total number of bytes written, and set
200 : * *<b>annotation_len_out</b> to the number of bytes written as
201 : * annotations. */
202 : static ssize_t
203 16 : dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
204 : {
205 16 : ssize_t r = 0;
206 16 : ssize_t written;
207 16 : if (md->body == NULL) {
208 0 : *annotation_len_out = 0;
209 0 : return 0;
210 : }
211 : /* XXXX drops unknown annotations. */
212 16 : if (md->last_listed) {
213 16 : char buf[ISO_TIME_LEN+1];
214 16 : char annotation[ISO_TIME_LEN+32];
215 16 : format_iso_time(buf, md->last_listed);
216 16 : tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
217 16 : if (write_all_to_fd(fd, annotation, strlen(annotation)) < 0) {
218 0 : log_warn(LD_DIR,
219 : "Couldn't write microdescriptor annotation: %s",
220 : strerror(errno));
221 0 : return -1;
222 : }
223 16 : r += strlen(annotation);
224 16 : *annotation_len_out = r;
225 : } else {
226 0 : *annotation_len_out = 0;
227 : }
228 :
229 16 : md->off = tor_fd_getpos(fd);
230 16 : warn_if_nul_found(md->body, md->bodylen, (int64_t) md->off,
231 : "dumping a microdescriptor");
232 16 : written = write_all_to_fd(fd, md->body, md->bodylen);
233 16 : if (written != (ssize_t)md->bodylen) {
234 0 : written = written < 0 ? 0 : written;
235 0 : log_warn(LD_DIR,
236 : "Couldn't dump microdescriptor (wrote %ld out of %lu): %s",
237 : (long)written, (unsigned long)md->bodylen,
238 : strerror(errno));
239 0 : return -1;
240 : }
241 16 : r += md->bodylen;
242 16 : return r;
243 : }
244 :
245 : /** Holds a pointer to the current microdesc_cache_t object, or NULL if no
246 : * such object has been allocated. */
247 : static microdesc_cache_t *the_microdesc_cache = NULL;
248 :
249 : /** Return a pointer to the microdescriptor cache, loading it if necessary. */
250 : microdesc_cache_t *
251 82 : get_microdesc_cache(void)
252 : {
253 82 : microdesc_cache_t *cache = get_microdesc_cache_noload();
254 82 : if (PREDICT_UNLIKELY(cache->is_loaded == 0)) {
255 18 : microdesc_cache_reload(cache);
256 : }
257 82 : return cache;
258 : }
259 :
260 : /** Return a pointer to the microdescriptor cache, creating (but not loading)
261 : * it if necessary. */
262 : static microdesc_cache_t *
263 82 : get_microdesc_cache_noload(void)
264 : {
265 82 : if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
266 18 : microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache));
267 18 : HT_INIT(microdesc_map, &cache->map);
268 18 : cache->cache_fname = get_cachedir_fname("cached-microdescs");
269 18 : cache->journal_fname = get_cachedir_fname("cached-microdescs.new");
270 18 : the_microdesc_cache = cache;
271 : }
272 82 : return the_microdesc_cache;
273 : }
274 :
275 : /* There are three sources of microdescriptors:
276 : 1) Generated by us while acting as a directory authority.
277 : 2) Loaded from the cache on disk.
278 : 3) Downloaded.
279 : */
280 :
281 : /** Decode the microdescriptors from the string starting at <b>s</b> and
282 : * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no_save</b>,
283 : * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE,
284 : * leave their bodies as pointers to the mmap'd cache. If where is
285 : * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is not -1,
286 : * set the last_listed field of every microdesc to listed_at. If
287 : * requested_digests is non-null, then it contains a list of digests we mean
288 : * to allow, so we should reject any non-requested microdesc with a different
289 : * digest, and alter the list to contain only the digests of those microdescs
290 : * we didn't find.
291 : * Return a newly allocated list of the added microdescriptors, or NULL */
292 : smartlist_t *
293 15 : microdescs_add_to_cache(microdesc_cache_t *cache,
294 : const char *s, const char *eos, saved_location_t where,
295 : int no_save, time_t listed_at,
296 : smartlist_t *requested_digests256)
297 : {
298 15 : void * const DIGEST_REQUESTED = (void*)1;
299 15 : void * const DIGEST_RECEIVED = (void*)2;
300 15 : void * const DIGEST_INVALID = (void*)3;
301 :
302 15 : smartlist_t *descriptors, *added;
303 15 : const int allow_annotations = (where != SAVED_NOWHERE);
304 15 : smartlist_t *invalid_digests = smartlist_new();
305 :
306 15 : descriptors = microdescs_parse_from_string(s, eos,
307 : allow_annotations,
308 : where, invalid_digests);
309 15 : if (listed_at != (time_t)-1) {
310 32 : SMARTLIST_FOREACH(descriptors, microdesc_t *, md,
311 : md->last_listed = listed_at);
312 : }
313 15 : if (requested_digests256) {
314 3 : digest256map_t *requested;
315 3 : requested = digest256map_new();
316 : /* Set requested[d] to DIGEST_REQUESTED for every md we requested. */
317 9 : SMARTLIST_FOREACH(requested_digests256, const uint8_t *, cp,
318 : digest256map_set(requested, cp, DIGEST_REQUESTED));
319 : /* Set requested[d] to DIGEST_INVALID for every md we requested which we
320 : * will never be able to parse. Remove the ones we didn't request from
321 : * invalid_digests.
322 : */
323 7 : SMARTLIST_FOREACH_BEGIN(invalid_digests, uint8_t *, cp) {
324 4 : if (digest256map_get(requested, cp)) {
325 2 : digest256map_set(requested, cp, DIGEST_INVALID);
326 : } else {
327 2 : tor_free(cp);
328 2 : SMARTLIST_DEL_CURRENT(invalid_digests, cp);
329 : }
330 4 : } SMARTLIST_FOREACH_END(cp);
331 : /* Update requested[d] to 2 for the mds we asked for and got. Delete the
332 : * ones we never requested from the 'descriptors' smartlist.
333 : */
334 19 : SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
335 16 : if (digest256map_get(requested, (const uint8_t*)md->digest)) {
336 3 : digest256map_set(requested, (const uint8_t*)md->digest,
337 : DIGEST_RECEIVED);
338 : } else {
339 13 : log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microdesc");
340 13 : microdesc_free(md);
341 13 : SMARTLIST_DEL_CURRENT(descriptors, md);
342 : }
343 16 : } SMARTLIST_FOREACH_END(md);
344 : /* Remove the ones we got or the invalid ones from requested_digests256.
345 : */
346 9 : SMARTLIST_FOREACH_BEGIN(requested_digests256, uint8_t *, cp) {
347 6 : void *status = digest256map_get(requested, cp);
348 6 : if (status == DIGEST_RECEIVED || status == DIGEST_INVALID) {
349 5 : tor_free(cp);
350 5 : SMARTLIST_DEL_CURRENT(requested_digests256, cp);
351 : }
352 6 : } SMARTLIST_FOREACH_END(cp);
353 3 : digest256map_free(requested, NULL);
354 : }
355 :
356 : /* For every requested microdescriptor that was unparseable, mark it
357 : * as not to be retried. */
358 15 : if (smartlist_len(invalid_digests)) {
359 6 : networkstatus_t *ns =
360 6 : networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
361 6 : if (ns) {
362 3 : SMARTLIST_FOREACH_BEGIN(invalid_digests, char *, d) {
363 2 : routerstatus_t *rs =
364 2 : router_get_mutable_consensus_status_by_descriptor_digest(ns, d);
365 2 : if (rs && tor_memeq(d, rs->descriptor_digest, DIGEST256_LEN)) {
366 2 : download_status_mark_impossible(&rs->dl_status);
367 : }
368 2 : } SMARTLIST_FOREACH_END(d);
369 : }
370 : }
371 22 : SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
372 15 : smartlist_free(invalid_digests);
373 :
374 15 : added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
375 15 : smartlist_free(descriptors);
376 15 : return added;
377 : }
378 :
379 : /** As microdescs_add_to_cache, but takes a list of microdescriptors instead of
380 : * a string to decode. Frees any members of <b>descriptors</b> that it does
381 : * not add. */
382 : smartlist_t *
383 15 : microdescs_add_list_to_cache(microdesc_cache_t *cache,
384 : smartlist_t *descriptors, saved_location_t where,
385 : int no_save)
386 : {
387 15 : smartlist_t *added;
388 15 : open_file_t *open_file = NULL;
389 15 : int fd = -1;
390 : // int n_added = 0;
391 15 : ssize_t size = 0;
392 :
393 15 : if (where == SAVED_NOWHERE && !no_save) {
394 9 : fd = start_writing_to_file(cache->journal_fname,
395 : OPEN_FLAGS_APPEND|O_BINARY,
396 : 0600, &open_file);
397 9 : if (fd < 0) {
398 0 : log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
399 : cache->journal_fname, strerror(errno));
400 : }
401 : }
402 :
403 15 : added = smartlist_new();
404 26 : SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
405 11 : microdesc_t *md2;
406 11 : md2 = HT_FIND(microdesc_map, &cache->map, md);
407 11 : if (md2) {
408 : /* We already had this one. */
409 0 : if (md2->last_listed < md->last_listed)
410 0 : md2->last_listed = md->last_listed;
411 0 : microdesc_free(md);
412 0 : if (where != SAVED_NOWHERE)
413 0 : cache->bytes_dropped += size;
414 0 : continue;
415 : }
416 :
417 : /* Okay, it's a new one. */
418 11 : if (fd >= 0) {
419 8 : size_t annotation_len;
420 8 : size = dump_microdescriptor(fd, md, &annotation_len);
421 8 : if (size < 0) {
422 : /* we already warned in dump_microdescriptor */
423 0 : abort_writing_to_file(open_file);
424 0 : fd = -1;
425 : } else {
426 8 : md->saved_location = SAVED_IN_JOURNAL;
427 8 : cache->journal_len += size;
428 : }
429 : } else {
430 3 : md->saved_location = where;
431 : }
432 :
433 11 : md->no_save = no_save;
434 :
435 11 : HT_INSERT(microdesc_map, &cache->map, md);
436 11 : md->held_in_map = 1;
437 11 : smartlist_add(added, md);
438 11 : ++cache->n_seen;
439 11 : cache->total_len_seen += md->bodylen;
440 11 : } SMARTLIST_FOREACH_END(md);
441 :
442 15 : if (fd >= 0) {
443 9 : if (finish_writing_to_file(open_file) < 0) {
444 0 : log_warn(LD_DIR, "Error appending to microdescriptor file: %s",
445 : strerror(errno));
446 0 : smartlist_clear(added);
447 0 : return added;
448 : }
449 : }
450 :
451 : {
452 15 : networkstatus_t *ns = networkstatus_get_latest_consensus();
453 15 : if (ns && ns->flavor == FLAV_MICRODESC)
454 0 : SMARTLIST_FOREACH(added, microdesc_t *, md, nodelist_add_microdesc(md));
455 : }
456 :
457 15 : if (smartlist_len(added))
458 8 : router_dir_info_changed();
459 :
460 : return added;
461 : }
462 :
463 : /** Remove every microdescriptor in <b>cache</b>. */
464 : void
465 24 : microdesc_cache_clear(microdesc_cache_t *cache)
466 : {
467 24 : microdesc_t **entry, **next;
468 :
469 34 : for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
470 10 : microdesc_t *md = *entry;
471 10 : next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
472 10 : md->held_in_map = 0;
473 10 : microdesc_free(md);
474 : }
475 24 : HT_CLEAR(microdesc_map, &cache->map);
476 24 : if (cache->cache_content) {
477 3 : int res = tor_munmap_file(cache->cache_content);
478 3 : if (res != 0) {
479 0 : log_warn(LD_FS,
480 : "tor_munmap_file() failed clearing microdesc cache; "
481 : "we are probably about to leak memory.");
482 : /* TODO something smarter? */
483 : }
484 3 : cache->cache_content = NULL;
485 : }
486 24 : cache->total_len_seen = 0;
487 24 : cache->n_seen = 0;
488 24 : cache->bytes_dropped = 0;
489 24 : }
490 :
491 : static void
492 19 : warn_if_nul_found(const char *inp, size_t len, int64_t offset,
493 : const char *activity)
494 : {
495 19 : const char *nul_found = memchr(inp, 0, len);
496 19 : if (BUG(nul_found)) {
497 0 : log_warn(LD_BUG, "Found unexpected NUL while %s, offset %"PRId64
498 : "at position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".",
499 : activity, offset, (nul_found - inp), len);
500 0 : const char *start_excerpt_at, *eos = inp + len;
501 0 : if ((nul_found - inp) >= 16)
502 0 : start_excerpt_at = nul_found - 16;
503 : else
504 : start_excerpt_at = inp;
505 0 : size_t excerpt_len = MIN(32, eos - start_excerpt_at);
506 0 : char tmp[65];
507 0 : base16_encode(tmp, sizeof(tmp), start_excerpt_at, excerpt_len);
508 0 : log_warn(LD_BUG, " surrounding string: %s", tmp);
509 : }
510 19 : }
511 :
512 : /** Reload the contents of <b>cache</b> from disk. If it is empty, load it
513 : * for the first time. Return 0 on success, -1 on failure. */
514 : int
515 18 : microdesc_cache_reload(microdesc_cache_t *cache)
516 : {
517 18 : struct stat st;
518 18 : char *journal_content;
519 18 : smartlist_t *added;
520 18 : tor_mmap_t *mm;
521 18 : int total = 0;
522 :
523 18 : microdesc_cache_clear(cache);
524 :
525 18 : cache->is_loaded = 1;
526 :
527 18 : mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
528 18 : if (mm) {
529 2 : warn_if_nul_found(mm->data, mm->size, 0, "scanning microdesc cache");
530 2 : added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
531 : SAVED_IN_CACHE, 0, -1, NULL);
532 2 : if (added) {
533 2 : total += smartlist_len(added);
534 2 : smartlist_free(added);
535 : }
536 : }
537 :
538 18 : journal_content = read_file_to_str(cache->journal_fname,
539 : RFTS_IGNORE_MISSING, &st);
540 18 : if (journal_content) {
541 1 : cache->journal_len = strlen(journal_content);
542 1 : warn_if_nul_found(journal_content, (size_t)st.st_size, 0,
543 : "reading microdesc journal");
544 2 : added = microdescs_add_to_cache(cache, journal_content,
545 1 : journal_content+st.st_size,
546 : SAVED_IN_JOURNAL, 0, -1, NULL);
547 1 : if (added) {
548 1 : total += smartlist_len(added);
549 1 : smartlist_free(added);
550 : }
551 1 : tor_free(journal_content);
552 : }
553 18 : log_info(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
554 : total);
555 :
556 18 : microdesc_cache_rebuild(cache, 0 /* don't force */);
557 :
558 18 : return 0;
559 : }
560 :
561 : /** By default, we remove any microdescriptors that have gone at least this
562 : * long without appearing in a current consensus. */
563 : #define TOLERATE_MICRODESC_AGE (7*24*60*60)
564 :
565 : /** Remove all microdescriptors from <b>cache</b> that haven't been listed for
566 : * a long time. Does not rebuild the cache on disk. If <b>cutoff</b> is
567 : * positive, specifically remove microdescriptors that have been unlisted
568 : * since <b>cutoff</b>. If <b>force</b> is true, remove microdescriptors even
569 : * if we have no current live microdescriptor consensus.
570 : */
571 : void
572 22 : microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
573 : {
574 22 : microdesc_t **mdp, *victim;
575 22 : int dropped=0, kept=0;
576 22 : size_t bytes_dropped = 0;
577 22 : time_t now = time(NULL);
578 :
579 : /* If we don't know a reasonably live consensus, don't believe last_listed
580 : * values: we might be starting up after being down for a while. */
581 43 : if (! force &&
582 21 : ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
583 : return;
584 :
585 8 : if (cutoff <= 0)
586 7 : cutoff = now - TOLERATE_MICRODESC_AGE;
587 :
588 11 : for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
589 3 : const int is_old = (*mdp)->last_listed < cutoff;
590 3 : const unsigned held_by_nodes = (*mdp)->held_by_nodes;
591 3 : if (is_old && !held_by_nodes) {
592 1 : ++dropped;
593 1 : victim = *mdp;
594 1 : mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
595 1 : victim->held_in_map = 0;
596 1 : bytes_dropped += victim->bodylen;
597 1 : microdesc_free(victim);
598 : } else {
599 2 : if (is_old) {
600 : /* It's old, but it has held_by_nodes set. That's not okay. */
601 : /* Let's try to diagnose and fix #7164 . */
602 0 : smartlist_t *nodes = nodelist_find_nodes_with_microdesc(*mdp);
603 0 : const networkstatus_t *ns = networkstatus_get_latest_consensus();
604 0 : long networkstatus_age = -1;
605 0 : const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
606 0 : if (ns) {
607 0 : networkstatus_age = now - ns->valid_after;
608 : }
609 0 : log_warn(LD_BUG, "Microdescriptor seemed very old "
610 : "(last listed %d hours ago vs %d hour cutoff), but is still "
611 : "marked as being held by %d node(s). I found %d node(s) "
612 : "holding it. Current networkstatus is %ld hours old. "
613 : "Hashtable badness is %d.",
614 : (int)((now - (*mdp)->last_listed) / 3600),
615 : (int)((now - cutoff) / 3600),
616 : held_by_nodes,
617 : smartlist_len(nodes),
618 : networkstatus_age / 3600,
619 : ht_badness);
620 :
621 0 : SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
622 0 : const char *rs_match = "No RS";
623 0 : const char *rs_present = "";
624 0 : if (node->rs) {
625 0 : if (tor_memeq(node->rs->descriptor_digest,
626 0 : (*mdp)->digest, DIGEST256_LEN)) {
627 : rs_match = "Microdesc digest in RS matches";
628 : } else {
629 0 : rs_match = "Microdesc digest in RS does match";
630 : }
631 0 : if (ns) {
632 : /* This should be impossible, but let's see! */
633 0 : rs_present = " RS not present in networkstatus.";
634 0 : SMARTLIST_FOREACH(ns->routerstatus_list, routerstatus_t *,rs, {
635 : if (rs == node->rs) {
636 : rs_present = " RS okay in networkstatus.";
637 : }
638 : });
639 : }
640 : }
641 0 : log_warn(LD_BUG, " [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s",
642 : node_sl_idx,
643 : hex_str(node->identity, DIGEST_LEN),
644 : node->md, node->rs, node->ri, rs_match, rs_present);
645 0 : } SMARTLIST_FOREACH_END(node);
646 0 : smartlist_free(nodes);
647 0 : (*mdp)->last_listed = now;
648 : }
649 :
650 2 : ++kept;
651 2 : mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
652 : }
653 : }
654 :
655 8 : if (dropped) {
656 1 : log_info(LD_DIR, "Removed %d/%d microdescriptors as old.",
657 : dropped,dropped+kept);
658 1 : cache->bytes_dropped += bytes_dropped;
659 : }
660 : }
661 :
662 : static int
663 18 : should_rebuild_md_cache(microdesc_cache_t *cache)
664 : {
665 36 : const size_t old_len =
666 18 : cache->cache_content ? cache->cache_content->size : 0;
667 18 : const size_t journal_len = cache->journal_len;
668 18 : const size_t dropped = cache->bytes_dropped;
669 :
670 18 : if (journal_len < 16384)
671 : return 0; /* Don't bother, not enough has happened yet. */
672 0 : if (dropped > (journal_len + old_len) / 3)
673 : return 1; /* We could save 1/3 or more of the currently used space. */
674 0 : if (journal_len > old_len / 2)
675 0 : return 1; /* We should append to the regular file */
676 :
677 : return 0;
678 : }
679 :
680 : /**
681 : * Mark <b>md</b> as having no body, and release any storage previously held
682 : * by its body.
683 : */
684 : static void
685 0 : microdesc_wipe_body(microdesc_t *md)
686 : {
687 0 : if (!md)
688 : return;
689 :
690 0 : if (md->saved_location != SAVED_IN_CACHE)
691 0 : tor_free(md->body);
692 :
693 0 : md->off = 0;
694 0 : md->saved_location = SAVED_NOWHERE;
695 0 : md->body = NULL;
696 0 : md->bodylen = 0;
697 0 : md->no_save = 1;
698 : }
699 :
700 : /** Regenerate the main cache file for <b>cache</b>, clear the journal file,
701 : * and update every microdesc_t in the cache with pointers to its new
702 : * location. If <b>force</b> is true, do this unconditionally. If
703 : * <b>force</b> is false, do it only if we expect to save space on disk. */
704 : int
705 21 : microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
706 : {
707 21 : open_file_t *open_file;
708 21 : int fd = -1, res;
709 21 : microdesc_t **mdp;
710 21 : smartlist_t *wrote;
711 21 : ssize_t size;
712 21 : off_t off = 0, off_real;
713 21 : int orig_size, new_size;
714 :
715 21 : if (cache == NULL) {
716 0 : cache = the_microdesc_cache;
717 0 : if (cache == NULL)
718 : return 0;
719 : }
720 :
721 : /* Remove dead descriptors */
722 21 : microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);
723 :
724 21 : if (!force && !should_rebuild_md_cache(cache))
725 : return 0;
726 :
727 3 : log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
728 :
729 3 : orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
730 3 : orig_size += (int)cache->journal_len;
731 :
732 3 : fd = start_writing_to_file(cache->cache_fname,
733 : OPEN_FLAGS_REPLACE|O_BINARY,
734 : 0600, &open_file);
735 3 : if (fd < 0)
736 : return -1;
737 :
738 3 : wrote = smartlist_new();
739 :
740 11 : HT_FOREACH(mdp, microdesc_map, &cache->map) {
741 8 : microdesc_t *md = *mdp;
742 8 : size_t annotation_len;
743 8 : if (md->no_save || !md->body)
744 0 : continue;
745 :
746 8 : size = dump_microdescriptor(fd, md, &annotation_len);
747 8 : if (size < 0) {
748 0 : microdesc_wipe_body(md);
749 :
750 : /* rewind, in case it was a partial write. */
751 0 : tor_fd_setpos(fd, off);
752 0 : continue;
753 : }
754 8 : tor_assert(((size_t)size) == annotation_len + md->bodylen);
755 8 : md->off = off + annotation_len;
756 8 : off += size;
757 8 : off_real = tor_fd_getpos(fd);
758 8 : if (off_real != off) {
759 0 : log_warn(LD_BUG, "Discontinuity in position in microdescriptor cache."
760 : "By my count, I'm at %"PRId64
761 : ", but I should be at %"PRId64,
762 : (int64_t)(off), (int64_t)(off_real));
763 0 : if (off_real >= 0)
764 0 : off = off_real;
765 : }
766 8 : if (md->saved_location != SAVED_IN_CACHE) {
767 4 : tor_free(md->body);
768 4 : md->saved_location = SAVED_IN_CACHE;
769 : }
770 8 : smartlist_add(wrote, md);
771 : }
772 :
773 : /* We must do this unmap _before_ we call finish_writing_to_file(), or
774 : * windows will not actually replace the file. */
775 3 : if (cache->cache_content) {
776 2 : res = tor_munmap_file(cache->cache_content);
777 2 : if (res != 0) {
778 0 : log_warn(LD_FS,
779 : "Failed to unmap old microdescriptor cache while rebuilding");
780 : }
781 2 : cache->cache_content = NULL;
782 : }
783 :
784 3 : if (finish_writing_to_file(open_file) < 0) {
785 0 : log_warn(LD_DIR, "Error rebuilding microdescriptor cache: %s",
786 : strerror(errno));
787 : /* Okay. Let's prevent from making things worse elsewhere. */
788 0 : cache->cache_content = NULL;
789 0 : HT_FOREACH(mdp, microdesc_map, &cache->map) {
790 0 : microdesc_t *md = *mdp;
791 0 : if (md->saved_location == SAVED_IN_CACHE) {
792 0 : microdesc_wipe_body(md);
793 : }
794 : }
795 0 : smartlist_free(wrote);
796 0 : return -1;
797 : }
798 :
799 3 : cache->cache_content = tor_mmap_file(cache->cache_fname);
800 :
801 3 : if (!cache->cache_content && smartlist_len(wrote)) {
802 0 : log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
803 : cache->cache_fname);
804 0 : smartlist_free(wrote);
805 0 : return -1;
806 : }
807 11 : SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
808 8 : tor_assert(md->saved_location == SAVED_IN_CACHE);
809 8 : md->body = (char*)cache->cache_content->data + md->off;
810 8 : if (PREDICT_UNLIKELY(
811 : md->bodylen < 9 || fast_memneq(md->body, "onion-key", 9) != 0)) {
812 : /* XXXX once bug 2022 is solved, we can kill this block and turn it
813 : * into just the tor_assert(fast_memeq) */
814 0 : off_t avail = cache->cache_content->size - md->off;
815 0 : char *bad_str;
816 0 : tor_assert(avail >= 0);
817 0 : bad_str = tor_strndup(md->body, MIN(128, (size_t)avail));
818 0 : log_err(LD_BUG, "After rebuilding microdesc cache, offsets seem wrong. "
819 : " At offset %d, I expected to find a microdescriptor starting "
820 : " with \"onion-key\". Instead I got %s.",
821 : (int)md->off, escaped(bad_str));
822 0 : tor_free(bad_str);
823 0 : tor_assert(fast_memeq(md->body, "onion-key", 9));
824 : }
825 8 : } SMARTLIST_FOREACH_END(md);
826 :
827 3 : smartlist_free(wrote);
828 :
829 3 : write_str_to_file(cache->journal_fname, "", 1);
830 3 : cache->journal_len = 0;
831 3 : cache->bytes_dropped = 0;
832 :
833 3 : new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
834 3 : log_info(LD_DIR, "Done rebuilding microdesc cache. "
835 : "Saved %d bytes; %d still used.",
836 : orig_size-new_size, new_size);
837 :
838 3 : return 0;
839 : }
840 :
841 : /** Make sure that the reference count of every microdescriptor in cache is
842 : * accurate. */
843 : void
844 0 : microdesc_check_counts(void)
845 : {
846 0 : microdesc_t **mdp;
847 0 : if (!the_microdesc_cache)
848 : return;
849 :
850 0 : HT_FOREACH(mdp, microdesc_map, &the_microdesc_cache->map) {
851 0 : microdesc_t *md = *mdp;
852 0 : unsigned int found=0;
853 0 : const smartlist_t *nodes = nodelist_get_list();
854 0 : SMARTLIST_FOREACH(nodes, node_t *, node, {
855 : if (node->md == md) {
856 : ++found;
857 : }
858 : });
859 0 : tor_assert(found == md->held_by_nodes);
860 : }
861 : }
862 :
863 : /** Deallocate a single microdescriptor. Note: the microdescriptor MUST have
864 : * previously been removed from the cache if it had ever been inserted. */
865 : void
866 103 : microdesc_free_(microdesc_t *md, const char *fname, int lineno)
867 : {
868 103 : if (!md)
869 : return;
870 :
871 : /* Make sure that the microdesc was really removed from the appropriate data
872 : structures. */
873 60 : if (md->held_in_map) {
874 0 : microdesc_cache_t *cache = get_microdesc_cache_noload();
875 0 : microdesc_t *md2 = HT_FIND(microdesc_map, &cache->map, md);
876 0 : if (md2 == md) {
877 0 : log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
878 : "in microdesc_map", fname, lineno);
879 0 : HT_REMOVE(microdesc_map, &cache->map, md);
880 : } else {
881 0 : log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_in_map "
882 : "set, but microdesc was not in the map.", fname, lineno);
883 : }
884 0 : tor_fragile_assert();
885 : }
886 60 : if (md->held_by_nodes) {
887 0 : microdesc_cache_t *cache = get_microdesc_cache_noload();
888 0 : int found=0;
889 0 : const smartlist_t *nodes = nodelist_get_list();
890 0 : const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
891 0 : SMARTLIST_FOREACH(nodes, node_t *, node, {
892 : if (node->md == md) {
893 : ++found;
894 : node->md = NULL;
895 : }
896 : });
897 0 : if (found) {
898 0 : log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
899 : "referenced %d node(s); held_by_nodes == %u, ht_badness == %d",
900 : fname, lineno, found, md->held_by_nodes, ht_badness);
901 : } else {
902 0 : log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_by_nodes "
903 : "set to %u, but md was not referenced by any nodes. "
904 : "ht_badness == %d",
905 : fname, lineno, md->held_by_nodes, ht_badness);
906 : }
907 0 : tor_fragile_assert();
908 : }
909 : //tor_assert(md->held_in_map == 0);
910 : //tor_assert(md->held_by_nodes == 0);
911 :
912 60 : if (md->onion_pkey)
913 46 : tor_free(md->onion_pkey);
914 60 : tor_free(md->onion_curve25519_pkey);
915 60 : tor_free(md->ed25519_identity_pkey);
916 60 : if (md->body && md->saved_location != SAVED_IN_CACHE)
917 49 : tor_free(md->body);
918 :
919 60 : nodefamily_free(md->family);
920 60 : short_policy_free(md->exit_policy);
921 60 : short_policy_free(md->ipv6_exit_policy);
922 :
923 60 : tor_free(md);
924 : }
925 :
926 : /** Free all storage held in the microdesc.c module. */
927 : void
928 241 : microdesc_free_all(void)
929 : {
930 241 : if (the_microdesc_cache) {
931 6 : microdesc_cache_clear(the_microdesc_cache);
932 6 : tor_free(the_microdesc_cache->cache_fname);
933 6 : tor_free(the_microdesc_cache->journal_fname);
934 6 : tor_free(the_microdesc_cache);
935 : }
936 :
937 241 : if (outdated_dirserver_list) {
938 0 : SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
939 0 : smartlist_free(outdated_dirserver_list);
940 : }
941 241 : }
942 :
943 : /** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
944 : * <b>d</b>, return it. Otherwise return NULL. */
945 : microdesc_t *
946 76 : microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
947 : {
948 76 : microdesc_t *md, search;
949 76 : if (!cache)
950 53 : cache = get_microdesc_cache();
951 76 : memcpy(search.digest, d, DIGEST256_LEN);
952 76 : md = HT_FIND(microdesc_map, &cache->map, &search);
953 76 : return md;
954 : }
955 :
956 : /** Return a smartlist of all the sha256 digest of the microdescriptors that
957 : * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
958 : * to internals of <b>ns</b>; you should not free the members of the resulting
959 : * smartlist. Omit all microdescriptors whose digest appear in <b>skip</b>. */
960 : smartlist_t *
961 1 : microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
962 : int downloadable_only, digest256map_t *skip)
963 : {
964 1 : smartlist_t *result = smartlist_new();
965 1 : time_t now = time(NULL);
966 1 : tor_assert(ns->flavor == FLAV_MICRODESC);
967 4 : SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
968 3 : if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
969 0 : continue;
970 6 : if (downloadable_only &&
971 3 : !download_status_is_ready(&rs->dl_status, now))
972 0 : continue;
973 3 : if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
974 0 : continue;
975 3 : if (fast_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
976 0 : continue;
977 : /* XXXX Also skip if we're a noncache and wouldn't use this router.
978 : * XXXX NM Microdesc
979 : */
980 3 : smartlist_add(result, rs->descriptor_digest);
981 3 : } SMARTLIST_FOREACH_END(rs);
982 1 : return result;
983 : }
984 :
985 : /** Launch download requests for microdescriptors as appropriate.
986 : *
987 : * Specifically, we should launch download requests if we are configured to
988 : * download mirodescriptors, and there are some microdescriptors listed in the
989 : * current microdesc consensus that we don't have, and either we never asked
990 : * for them, or we failed to download them but we're willing to retry.
991 : */
992 : void
993 1 : update_microdesc_downloads(time_t now)
994 : {
995 1 : const or_options_t *options = get_options();
996 1 : networkstatus_t *consensus;
997 1 : smartlist_t *missing;
998 1 : digest256map_t *pending;
999 :
1000 1 : if (should_delay_dir_fetches(options, NULL))
1001 1 : return;
1002 1 : if (dirclient_too_idle_to_fetch_descriptors(options, now))
1003 : return;
1004 :
1005 : /* Give up if we don't have a reasonably live consensus. */
1006 1 : consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
1007 1 : if (!consensus)
1008 : return;
1009 :
1010 1 : if (!we_fetch_microdescriptors(options))
1011 : return;
1012 :
1013 1 : pending = digest256map_new();
1014 1 : list_pending_microdesc_downloads(pending);
1015 :
1016 1 : missing = microdesc_list_missing_digest256(consensus,
1017 : get_microdesc_cache(),
1018 : 1,
1019 : pending);
1020 1 : digest256map_free(pending, NULL);
1021 :
1022 1 : launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC,
1023 : missing, NULL, now);
1024 :
1025 1 : smartlist_free(missing);
1026 : }
1027 :
1028 : /** For every microdescriptor listed in the current microdescriptor consensus,
1029 : * update its last_listed field to be at least as recent as the publication
1030 : * time of the current microdescriptor consensus.
1031 : */
1032 : void
1033 1 : update_microdescs_from_networkstatus(time_t now)
1034 : {
1035 1 : microdesc_cache_t *cache = get_microdesc_cache();
1036 1 : microdesc_t *md;
1037 1 : networkstatus_t *ns =
1038 1 : networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
1039 :
1040 1 : if (! ns)
1041 : return;
1042 :
1043 1 : tor_assert(ns->flavor == FLAV_MICRODESC);
1044 :
1045 4 : SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
1046 3 : md = microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest);
1047 3 : if (md && ns->valid_after > md->last_listed)
1048 0 : md->last_listed = ns->valid_after;
1049 3 : } SMARTLIST_FOREACH_END(rs);
1050 : }
1051 :
1052 : /** Return true iff we should prefer to use microdescriptors rather than
1053 : * routerdescs for building circuits. */
1054 : int
1055 154063 : we_use_microdescriptors_for_circuits(const or_options_t *options)
1056 : {
1057 154063 : if (options->UseMicrodescriptors == 0)
1058 128 : return 0; /* the user explicitly picked no */
1059 : return 1; /* yes and auto both mean yes */
1060 : }
1061 :
1062 : /** Return true iff we should try to download microdescriptors at all. */
1063 : int
1064 1 : we_fetch_microdescriptors(const or_options_t *options)
1065 : {
1066 1 : if (directory_caches_dir_info(options))
1067 : return 1;
1068 1 : if (options->FetchUselessDescriptors)
1069 : return 1;
1070 1 : return we_use_microdescriptors_for_circuits(options);
1071 : }
1072 :
1073 : /** Return true iff we should try to download router descriptors at all. */
1074 : int
1075 1 : we_fetch_router_descriptors(const or_options_t *options)
1076 : {
1077 1 : if (directory_caches_dir_info(options))
1078 : return 1;
1079 1 : if (options->FetchUselessDescriptors)
1080 : return 1;
1081 1 : return ! we_use_microdescriptors_for_circuits(options);
1082 : }
1083 :
1084 : /** Return the consensus flavor we actually want to use to build circuits. */
1085 103543 : MOCK_IMPL(int,
1086 : usable_consensus_flavor,(void))
1087 : {
1088 103543 : if (we_use_microdescriptors_for_circuits(get_options())) {
1089 : return FLAV_MICRODESC;
1090 : } else {
1091 17 : return FLAV_NS;
1092 : }
1093 : }
|
