Line data Source code
1 : /* Copyright (c) 2017-2021, The Tor Project, Inc. */
2 : /* See LICENSE for licensing information */
3 :
4 : /**
5 : * \file consdiffmgr.c
6 : *
7 : * \brief consensus diff manager functions
8 : *
9 : * This module is run by directory authorities and caches in order
10 : * to remember a number of past consensus documents, and to generate
11 : * and serve the diffs from those documents to the latest consensus.
12 : */
13 :
14 : #define CONSDIFFMGR_PRIVATE
15 :
16 : #include "core/or/or.h"
17 : #include "app/config/config.h"
18 : #include "feature/dircache/conscache.h"
19 : #include "feature/dircommon/consdiff.h"
20 : #include "feature/dircache/consdiffmgr.h"
21 : #include "core/mainloop/cpuworker.h"
22 : #include "feature/nodelist/networkstatus.h"
23 : #include "feature/dirparse/ns_parse.h"
24 : #include "lib/evloop/compat_libevent.h"
25 : #include "lib/evloop/workqueue.h"
26 : #include "lib/compress/compress.h"
27 : #include "lib/encoding/confline.h"
28 :
29 : #include "feature/nodelist/networkstatus_st.h"
30 : #include "feature/nodelist/networkstatus_voter_info_st.h"
31 :
32 : /**
33 : * Labels to apply to items in the conscache object.
34 : *
35 : * @{
36 : */
37 : /* One of DOCTYPE_CONSENSUS or DOCTYPE_CONSENSUS_DIFF */
38 : #define LABEL_DOCTYPE "document-type"
39 : /* The valid-after time for a consensus (or for the target consensus of a
40 : * diff), encoded as ISO UTC. */
41 : #define LABEL_VALID_AFTER "consensus-valid-after"
42 : /* The fresh-until time for a consensus (or for the target consensus of a
43 : * diff), encoded as ISO UTC. */
44 : #define LABEL_FRESH_UNTIL "consensus-fresh-until"
45 : /* The valid-until time for a consensus (or for the target consensus of a
46 : * diff), encoded as ISO UTC. */
47 : #define LABEL_VALID_UNTIL "consensus-valid-until"
48 : /* Comma-separated list of hex-encoded identity digests for the voting
49 : * authorities. */
50 : #define LABEL_SIGNATORIES "consensus-signatories"
51 : /* A hex encoded SHA3 digest of the object, as compressed (if any) */
52 : #define LABEL_SHA3_DIGEST "sha3-digest"
53 : /* A hex encoded SHA3 digest of the object before compression. */
54 : #define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed"
55 : /* A hex encoded SHA3 digest-as-signed of a consensus */
56 : #define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed"
57 : /* The flavor of the consensus or consensuses diff */
58 : #define LABEL_FLAVOR "consensus-flavor"
59 : /* Diff only: the SHA3 digest-as-signed of the source consensus. */
60 : #define LABEL_FROM_SHA3_DIGEST "from-sha3-digest"
61 : /* Diff only: the SHA3 digest-in-full of the target consensus. */
62 : #define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest"
63 : /* Diff only: the valid-after date of the source consensus. */
64 : #define LABEL_FROM_VALID_AFTER "from-valid-after"
65 : /* What kind of compression was used? */
66 : #define LABEL_COMPRESSION_TYPE "compression"
67 : /** @} */
68 :
69 : #define DOCTYPE_CONSENSUS "consensus"
70 : #define DOCTYPE_CONSENSUS_DIFF "consensus-diff"
71 :
72 : /**
73 : * Underlying directory that stores consensuses and consensus diffs. Don't
74 : * use this directly: use cdm_cache_get() instead.
75 : */
76 : static consensus_cache_t *cons_diff_cache = NULL;
77 : /**
78 : * If true, we have learned at least one new consensus since the
79 : * consensus cache was last up-to-date.
80 : */
81 : static int cdm_cache_dirty = 0;
82 : /**
83 : * If true, we have scanned the cache to update our hashtable of diffs.
84 : */
85 : static int cdm_cache_loaded = 0;
86 :
87 : /**
88 : * Possible status values for cdm_diff_t.cdm_diff_status
89 : **/
90 : typedef enum cdm_diff_status_t {
91 : CDM_DIFF_PRESENT=1,
92 : CDM_DIFF_IN_PROGRESS=2,
93 : CDM_DIFF_ERROR=3,
94 : } cdm_diff_status_t;
95 :
96 : /** Which methods do we use for precompressing diffs? */
97 : static const compress_method_t compress_diffs_with[] = {
98 : NO_METHOD,
99 : GZIP_METHOD,
100 : #ifdef HAVE_LZMA
101 : LZMA_METHOD,
102 : #endif
103 : #ifdef HAVE_ZSTD
104 : ZSTD_METHOD,
105 : #endif
106 : };
107 :
108 : /**
109 : * Event for rescanning the cache.
110 : */
111 : static mainloop_event_t *consdiffmgr_rescan_ev = NULL;
112 :
113 : static void consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg);
114 : static void mark_cdm_cache_dirty(void);
115 :
116 : /** How many different methods will we try to use for diff compression? */
117 : STATIC unsigned
118 164 : n_diff_compression_methods(void)
119 : {
120 164 : return ARRAY_LENGTH(compress_diffs_with);
121 : }
122 :
123 : /** Which methods do we use for precompressing consensuses? */
124 : static const compress_method_t compress_consensus_with[] = {
125 : ZLIB_METHOD,
126 : #ifdef HAVE_LZMA
127 : LZMA_METHOD,
128 : #endif
129 : #ifdef HAVE_ZSTD
130 : ZSTD_METHOD,
131 : #endif
132 : };
133 :
134 : /** How many different methods will we try to use for diff compression? */
135 : STATIC unsigned
136 2219 : n_consensus_compression_methods(void)
137 : {
138 2219 : return ARRAY_LENGTH(compress_consensus_with);
139 : }
140 :
141 : /** For which compression method do we retain old consensuses? There's no
142 : * need to keep all of them, since we won't be serving them. We'll
143 : * go with ZLIB_METHOD because it's pretty fast and everyone has it.
144 : */
145 : #define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD
146 :
147 : /** Handles pointing to the latest consensus entries as compressed and
148 : * stored. */
149 : static consensus_cache_entry_handle_t *
150 : latest_consensus[N_CONSENSUS_FLAVORS]
151 : [ARRAY_LENGTH(compress_consensus_with)];
152 :
153 : /** Hashtable node used to remember the current status of the diff
154 : * from a given sha3 digest to the current consensus. */
155 : typedef struct cdm_diff_t {
156 : HT_ENTRY(cdm_diff_t) node;
157 :
158 : /** Consensus flavor for this diff (part of ht key) */
159 : consensus_flavor_t flavor;
160 : /** SHA3-256 digest of the consensus that this diff is _from_. (part of the
161 : * ht key) */
162 : uint8_t from_sha3[DIGEST256_LEN];
163 : /** Method by which the diff is compressed. (part of the ht key */
164 : compress_method_t compress_method;
165 :
166 : /** One of the CDM_DIFF_* values, depending on whether this diff
167 : * is available, in progress, or impossible to compute. */
168 : cdm_diff_status_t cdm_diff_status;
169 : /** SHA3-256 digest of the consensus that this diff is _to. */
170 : uint8_t target_sha3[DIGEST256_LEN];
171 :
172 : /** Handle to the cache entry for this diff, if any. We use a handle here
173 : * to avoid thinking too hard about cache entry lifetime issues. */
174 : consensus_cache_entry_handle_t *entry;
175 : } cdm_diff_t;
176 :
177 : /** Hashtable mapping flavor and source consensus digest to status. */
178 : static HT_HEAD(cdm_diff_ht, cdm_diff_t) cdm_diff_ht = HT_INITIALIZER();
179 :
180 : #ifdef _WIN32
181 : // XXX(ahf): For tor#24857, a contributor suggested that on Windows, the CPU
182 : // begins to spike at 100% once the number of files handled by the consensus
183 : // diff manager becomes larger than 64. To see if the issue goes away, we
184 : // hardcode this value to 64 now while we investigate a better solution.
185 : # define CACHE_MAX_NUM 64
186 : #else /* !defined(_WIN32) */
187 : # define CACHE_MAX_NUM 128
188 : #endif /* defined(_WIN32) */
189 :
190 : /**
191 : * Configuration for this module
192 : */
193 : static consdiff_cfg_t consdiff_cfg = {
194 : // XXXX I'd like to make this number bigger, but it interferes with the
195 : // XXXX seccomp2 syscall filter, which tops out at BPF_MAXINS (4096)
196 : // XXXX rules.
197 : /* .cache_max_num = */ CACHE_MAX_NUM
198 : };
199 :
200 : static int consdiffmgr_ensure_space_for_files(int n);
201 : static int consensus_queue_compression_work(const char *consensus,
202 : size_t consensus_len,
203 : const networkstatus_t *as_parsed);
204 : static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
205 : consensus_cache_entry_t *diff_to);
206 : static void consdiffmgr_set_cache_flags(void);
207 :
208 : /* =====
209 : * Hashtable setup
210 : * ===== */
211 :
212 : /** Helper: hash the key of a cdm_diff_t. */
213 : static unsigned
214 251 : cdm_diff_hash(const cdm_diff_t *diff)
215 : {
216 251 : uint8_t tmp[DIGEST256_LEN + 2];
217 251 : memcpy(tmp, diff->from_sha3, DIGEST256_LEN);
218 251 : tmp[DIGEST256_LEN] = (uint8_t) diff->flavor;
219 251 : tmp[DIGEST256_LEN+1] = (uint8_t) diff->compress_method;
220 251 : return (unsigned) siphash24g(tmp, sizeof(tmp));
221 : }
222 : /** Helper: compare two cdm_diff_t objects for key equality */
223 : static int
224 99 : cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2)
225 : {
226 181 : return fast_memeq(diff1->from_sha3, diff2->from_sha3, DIGEST256_LEN) &&
227 99 : diff1->flavor == diff2->flavor &&
228 82 : diff1->compress_method == diff2->compress_method;
229 : }
230 :
231 1485 : HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq);
232 5 : HT_GENERATE2(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq,
233 : 0.6, tor_reallocarray, tor_free_);
234 :
235 : #define cdm_diff_free(diff) \
236 : FREE_AND_NULL(cdm_diff_t, cdm_diff_free_, (diff))
237 :
238 : /** Release all storage held in <b>diff</b>. */
239 : static void
240 84 : cdm_diff_free_(cdm_diff_t *diff)
241 : {
242 84 : if (!diff)
243 : return;
244 84 : consensus_cache_entry_handle_free(diff->entry);
245 84 : tor_free(diff);
246 : }
247 :
248 : /** Create and return a new cdm_diff_t with the given values. Does not
249 : * add it to the hashtable. */
250 : static cdm_diff_t *
251 84 : cdm_diff_new(consensus_flavor_t flav,
252 : const uint8_t *from_sha3,
253 : const uint8_t *target_sha3,
254 : compress_method_t method)
255 : {
256 84 : cdm_diff_t *ent;
257 84 : ent = tor_malloc_zero(sizeof(cdm_diff_t));
258 84 : ent->flavor = flav;
259 84 : memcpy(ent->from_sha3, from_sha3, DIGEST256_LEN);
260 84 : memcpy(ent->target_sha3, target_sha3, DIGEST256_LEN);
261 84 : ent->compress_method = method;
262 84 : return ent;
263 : }
264 :
265 : /**
266 : * Examine the diff hashtable to see whether we know anything about computing
267 : * a diff of type <b>flav</b> between consensuses with the two provided
268 : * SHA3-256 digests. If a computation is in progress, or if the computation
269 : * has already been tried and failed, return 1. Otherwise, note the
270 : * computation as "in progress" so that we don't reattempt it later, and
271 : * return 0.
272 : */
273 : static int
274 16 : cdm_diff_ht_check_and_note_pending(consensus_flavor_t flav,
275 : const uint8_t *from_sha3,
276 : const uint8_t *target_sha3)
277 : {
278 16 : struct cdm_diff_t search, *ent;
279 16 : unsigned u;
280 16 : int result = 0;
281 80 : for (u = 0; u < n_diff_compression_methods(); ++u) {
282 64 : compress_method_t method = compress_diffs_with[u];
283 64 : memset(&search, 0, sizeof(cdm_diff_t));
284 64 : search.flavor = flav;
285 64 : search.compress_method = method;
286 64 : memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
287 64 : ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
288 64 : if (ent) {
289 4 : tor_assert_nonfatal(ent->cdm_diff_status != CDM_DIFF_PRESENT);
290 4 : result = 1;
291 4 : continue;
292 : }
293 60 : ent = cdm_diff_new(flav, from_sha3, target_sha3, method);
294 60 : ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
295 60 : HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
296 : }
297 16 : return result;
298 : }
299 :
300 : /**
301 : * Update the status of the diff of type <b>flav</b> between consensuses with
302 : * the two provided SHA3-256 digests, so that its status becomes
303 : * <b>status</b>, and its value becomes the <b>handle</b>. If <b>handle</b>
304 : * is NULL, then the old handle (if any) is freed, and replaced with NULL.
305 : */
306 : static void
307 72 : cdm_diff_ht_set_status(consensus_flavor_t flav,
308 : const uint8_t *from_sha3,
309 : const uint8_t *to_sha3,
310 : compress_method_t method,
311 : int status,
312 : consensus_cache_entry_handle_t *handle)
313 : {
314 72 : if (handle == NULL) {
315 4 : tor_assert_nonfatal(status != CDM_DIFF_PRESENT);
316 : }
317 :
318 72 : struct cdm_diff_t search, *ent;
319 72 : memset(&search, 0, sizeof(cdm_diff_t));
320 72 : search.flavor = flav;
321 72 : search.compress_method = method,
322 72 : memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
323 72 : ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
324 72 : if (!ent) {
325 24 : ent = cdm_diff_new(flav, from_sha3, to_sha3, method);
326 24 : ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
327 24 : HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
328 48 : } else if (fast_memneq(ent->target_sha3, to_sha3, DIGEST256_LEN)) {
329 : // This can happen under certain really pathological conditions
330 : // if we decide we don't care about a diff before it is actually
331 : // done computing.
332 0 : return;
333 : }
334 :
335 72 : tor_assert_nonfatal(ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS);
336 :
337 72 : ent->cdm_diff_status = status;
338 72 : consensus_cache_entry_handle_free(ent->entry);
339 72 : ent->entry = handle;
340 : }
341 :
342 : /**
343 : * Helper: Remove from the hash table every present (actually computed) diff
344 : * of type <b>flav</b> whose target digest does not match
345 : * <b>unless_target_sha3_matches</b>.
346 : *
347 : * This function is used for the hash table to throw away references to diffs
348 : * that do not lead to the most given consensus of a given flavor.
349 : */
350 : static void
351 16 : cdm_diff_ht_purge(consensus_flavor_t flav,
352 : const uint8_t *unless_target_sha3_matches)
353 : {
354 16 : cdm_diff_t **diff, **next;
355 16 : for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
356 136 : cdm_diff_t *this = *diff;
357 :
358 136 : if ((*diff)->cdm_diff_status == CDM_DIFF_PRESENT &&
359 120 : flav == (*diff)->flavor) {
360 :
361 128 : if (BUG((*diff)->entry == NULL) ||
362 64 : consensus_cache_entry_handle_get((*diff)->entry) == NULL) {
363 : /* the underlying entry has gone away; drop this. */
364 7 : next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
365 7 : cdm_diff_free(this);
366 159 : continue;
367 : }
368 :
369 57 : if (unless_target_sha3_matches &&
370 57 : fast_memneq(unless_target_sha3_matches, (*diff)->target_sha3,
371 : DIGEST256_LEN)) {
372 : /* target hash doesn't match; drop this. */
373 1 : next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
374 1 : cdm_diff_free(this);
375 1 : continue;
376 : }
377 : }
378 128 : next = HT_NEXT(cdm_diff_ht, &cdm_diff_ht, diff);
379 : }
380 16 : }
381 :
382 : /**
383 : * Helper: initialize <b>cons_diff_cache</b>.
384 : */
385 : static void
386 21 : cdm_cache_init(void)
387 : {
388 21 : unsigned n_entries = consdiff_cfg.cache_max_num * 2;
389 :
390 21 : tor_assert(cons_diff_cache == NULL);
391 21 : cons_diff_cache = consensus_cache_open("diff-cache", n_entries);
392 21 : if (cons_diff_cache == NULL) {
393 : // LCOV_EXCL_START
394 : log_err(LD_FS, "Error: Couldn't open storage for consensus diffs.");
395 : tor_assert_unreached();
396 : // LCOV_EXCL_STOP
397 : } else {
398 21 : consdiffmgr_set_cache_flags();
399 : }
400 42 : consdiffmgr_rescan_ev =
401 21 : mainloop_event_postloop_new(consdiffmgr_rescan_cb, NULL);
402 21 : mark_cdm_cache_dirty();
403 21 : cdm_cache_loaded = 0;
404 21 : }
405 :
406 : /**
407 : * Helper: return the consensus_cache_t * that backs this manager,
408 : * initializing it if needed.
409 : */
410 : STATIC consensus_cache_t *
411 439 : cdm_cache_get(void)
412 : {
413 439 : if (PREDICT_UNLIKELY(cons_diff_cache == NULL)) {
414 21 : cdm_cache_init();
415 : }
416 439 : return cons_diff_cache;
417 : }
418 :
419 : /**
420 : * Helper: given a list of labels, prepend the hex-encoded SHA3 digest
421 : * of the <b>bodylen</b>-byte object at <b>body</b> to those labels,
422 : * with <b>label</b> as its label.
423 : */
424 : static void
425 195 : cdm_labels_prepend_sha3(config_line_t **labels,
426 : const char *label,
427 : const uint8_t *body,
428 : size_t bodylen)
429 : {
430 195 : uint8_t sha3_digest[DIGEST256_LEN];
431 195 : char hexdigest[HEX_DIGEST256_LEN+1];
432 195 : crypto_digest256((char *)sha3_digest,
433 : (const char *)body, bodylen, DIGEST_SHA3_256);
434 195 : base16_encode(hexdigest, sizeof(hexdigest),
435 : (const char *)sha3_digest, sizeof(sha3_digest));
436 :
437 195 : config_line_prepend(labels, label, hexdigest);
438 195 : }
439 :
440 : /** Helper: if there is a sha3-256 hex-encoded digest in <b>ent</b> with the
441 : * given label, set <b>digest_out</b> to that value (decoded), and return 0.
442 : *
443 : * Return -1 if there is no such label, and -2 if it is badly formatted. */
444 : STATIC int
445 120 : cdm_entry_get_sha3_value(uint8_t *digest_out,
446 : consensus_cache_entry_t *ent,
447 : const char *label)
448 : {
449 120 : if (ent == NULL)
450 : return -1;
451 :
452 119 : const char *hex = consensus_cache_entry_get_value(ent, label);
453 119 : if (hex == NULL)
454 : return -1;
455 :
456 117 : int n = base16_decode((char*)digest_out, DIGEST256_LEN, hex, strlen(hex));
457 117 : if (n != DIGEST256_LEN)
458 : return -2;
459 : else
460 113 : return 0;
461 : }
462 :
463 : /**
464 : * Helper: look for a consensus with the given <b>flavor</b> and
465 : * <b>valid_after</b> time in the cache. Return that consensus if it's
466 : * present, or NULL if it's missing.
467 : */
468 : STATIC consensus_cache_entry_t *
469 32 : cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after)
470 : {
471 32 : char formatted_time[ISO_TIME_LEN+1];
472 32 : format_iso_time_nospace(formatted_time, valid_after);
473 32 : const char *flavname = networkstatus_get_flavor_name(flavor);
474 :
475 : /* We'll filter by valid-after time first, since that should
476 : * match the fewest documents. */
477 : /* We could add an extra hashtable here, but since we only do this scan
478 : * when adding a new consensus, it probably doesn't matter much. */
479 32 : smartlist_t *matches = smartlist_new();
480 32 : consensus_cache_find_all(matches, cdm_cache_get(),
481 : LABEL_VALID_AFTER, formatted_time);
482 32 : consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
483 32 : consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
484 :
485 32 : consensus_cache_entry_t *result = NULL;
486 32 : if (smartlist_len(matches)) {
487 2 : result = smartlist_get(matches, 0);
488 : }
489 32 : smartlist_free(matches);
490 :
491 32 : return result;
492 : }
493 :
494 : /** Return the maximum age (in seconds) of consensuses that we should consider
495 : * storing. The available space in the directory may impose additional limits
496 : * on how much we store. */
497 : static int32_t
498 48 : get_max_age_to_cache(void)
499 : {
500 48 : const int32_t DEFAULT_MAX_AGE_TO_CACHE = 8192;
501 48 : const int32_t MIN_MAX_AGE_TO_CACHE = 0;
502 48 : const int32_t MAX_MAX_AGE_TO_CACHE = 8192;
503 48 : const char MAX_AGE_TO_CACHE_NAME[] = "max-consensus-age-to-cache-for-diff";
504 :
505 48 : const or_options_t *options = get_options();
506 :
507 48 : if (options->MaxConsensusAgeForDiffs) {
508 0 : const int v = options->MaxConsensusAgeForDiffs;
509 0 : if (v >= MAX_MAX_AGE_TO_CACHE * 3600)
510 : return MAX_MAX_AGE_TO_CACHE;
511 : else
512 0 : return v;
513 : }
514 :
515 : /* The parameter is in hours, so we multiply */
516 48 : return 3600 * networkstatus_get_param(NULL,
517 : MAX_AGE_TO_CACHE_NAME,
518 : DEFAULT_MAX_AGE_TO_CACHE,
519 : MIN_MAX_AGE_TO_CACHE,
520 : MAX_MAX_AGE_TO_CACHE);
521 : }
522 :
523 : #ifdef TOR_UNIT_TESTS
524 : /** As consdiffmgr_add_consensus, but requires a nul-terminated input. For
525 : * testing. */
526 : int
527 30 : consdiffmgr_add_consensus_nulterm(const char *consensus,
528 : const networkstatus_t *as_parsed)
529 : {
530 30 : size_t len = strlen(consensus);
531 : /* make a non-nul-terminated copy so that we can have a better chance
532 : * of catching errors. */
533 30 : char *ctmp = tor_memdup(consensus, len);
534 30 : int r = consdiffmgr_add_consensus(ctmp, len, as_parsed);
535 30 : tor_free(ctmp);
536 30 : return r;
537 : }
538 : #endif /* defined(TOR_UNIT_TESTS) */
539 :
540 : /**
541 : * Given a buffer containing a networkstatus consensus, and the results of
542 : * having parsed that consensus, add that consensus to the cache if it is not
543 : * already present and not too old. Create new consensus diffs from or to
544 : * that consensus as appropriate.
545 : *
546 : * Return 0 on success and -1 on failure.
547 : */
548 : int
549 30 : consdiffmgr_add_consensus(const char *consensus,
550 : size_t consensus_len,
551 : const networkstatus_t *as_parsed)
552 : {
553 30 : if (BUG(consensus == NULL) || BUG(as_parsed == NULL))
554 : return -1; // LCOV_EXCL_LINE
555 30 : if (BUG(as_parsed->type != NS_TYPE_CONSENSUS))
556 : return -1; // LCOV_EXCL_LINE
557 :
558 30 : const consensus_flavor_t flavor = as_parsed->flavor;
559 30 : const time_t valid_after = as_parsed->valid_after;
560 :
561 30 : if (valid_after < approx_time() - get_max_age_to_cache()) {
562 1 : log_info(LD_DIRSERV, "We don't care about this consensus document; it's "
563 : "too old.");
564 1 : return -1;
565 : }
566 :
567 : /* Do we already have this one? */
568 29 : consensus_cache_entry_t *entry =
569 29 : cdm_cache_lookup_consensus(flavor, valid_after);
570 29 : if (entry) {
571 1 : log_info(LD_DIRSERV, "We already have a copy of that consensus");
572 1 : return -1;
573 : }
574 :
575 : /* We don't have it. Add it to the cache. */
576 28 : return consensus_queue_compression_work(consensus, consensus_len, as_parsed);
577 : }
578 :
579 : /**
580 : * Helper: used to sort two smartlists of consensus_cache_entry_t by their
581 : * LABEL_VALID_AFTER labels.
582 : */
583 : static int
584 423 : compare_by_valid_after_(const void **a, const void **b)
585 : {
586 423 : const consensus_cache_entry_t *e1 = *a;
587 423 : const consensus_cache_entry_t *e2 = *b;
588 : /* We're in luck here: sorting UTC iso-encoded values lexically will work
589 : * fine (until 9999). */
590 423 : return strcmp_opt(consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER),
591 : consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER));
592 : }
593 :
594 : /**
595 : * Helper: Sort <b>lst</b> by LABEL_VALID_AFTER and return the most recent
596 : * entry.
597 : */
598 : static consensus_cache_entry_t *
599 112 : sort_and_find_most_recent(smartlist_t *lst)
600 : {
601 112 : smartlist_sort(lst, compare_by_valid_after_);
602 112 : if (smartlist_len(lst)) {
603 58 : return smartlist_get(lst, smartlist_len(lst) - 1);
604 : } else {
605 : return NULL;
606 : }
607 : }
608 :
609 : /** Return i such that compress_consensus_with[i] == method. Return
610 : * -1 if no such i exists. */
611 : static int
612 30 : consensus_compression_method_pos(compress_method_t method)
613 : {
614 30 : unsigned i;
615 66 : for (i = 0; i < n_consensus_compression_methods(); ++i) {
616 60 : if (compress_consensus_with[i] == method) {
617 24 : return i;
618 : }
619 : }
620 : return -1;
621 : }
622 :
623 : /**
624 : * If we know a consensus with the flavor <b>flavor</b> compressed with
625 : * <b>method</b>, set *<b>entry_out</b> to that value. Return values are as
626 : * for consdiffmgr_find_diff_from().
627 : */
628 : consdiff_status_t
629 12 : consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out,
630 : consensus_flavor_t flavor,
631 : compress_method_t method)
632 : {
633 12 : tor_assert(entry_out);
634 12 : tor_assert((int)flavor < N_CONSENSUS_FLAVORS);
635 :
636 12 : int pos = consensus_compression_method_pos(method);
637 12 : if (pos < 0) {
638 : // We don't compress consensuses with this method.
639 : return CONSDIFF_NOT_FOUND;
640 : }
641 6 : consensus_cache_entry_handle_t *handle = latest_consensus[flavor][pos];
642 6 : if (!handle)
643 : return CONSDIFF_NOT_FOUND;
644 5 : *entry_out = consensus_cache_entry_handle_get(handle);
645 5 : if (*entry_out)
646 : return CONSDIFF_AVAILABLE;
647 : else
648 0 : return CONSDIFF_NOT_FOUND;
649 : }
650 :
651 : /**
652 : * Look up consensus_cache_entry_t for the consensus of type <b>flavor</b>,
653 : * from the source consensus with the specified digest (which must be SHA3).
654 : *
655 : * If the diff is present, store it into *<b>entry_out</b> and return
656 : * CONSDIFF_AVAILABLE. Otherwise return CONSDIFF_NOT_FOUND or
657 : * CONSDIFF_IN_PROGRESS.
658 : */
659 : consdiff_status_t
660 31 : consdiffmgr_find_diff_from(consensus_cache_entry_t **entry_out,
661 : consensus_flavor_t flavor,
662 : int digest_type,
663 : const uint8_t *digest,
664 : size_t digestlen,
665 : compress_method_t method)
666 : {
667 31 : if (BUG(digest_type != DIGEST_SHA3_256) ||
668 31 : BUG(digestlen != DIGEST256_LEN)) {
669 : return CONSDIFF_NOT_FOUND; // LCOV_EXCL_LINE
670 : }
671 :
672 : // Try to look up the entry in the hashtable.
673 31 : cdm_diff_t search, *ent;
674 31 : memset(&search, 0, sizeof(search));
675 31 : search.flavor = flavor;
676 31 : search.compress_method = method;
677 31 : memcpy(search.from_sha3, digest, DIGEST256_LEN);
678 31 : ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
679 :
680 31 : if (ent == NULL ||
681 27 : ent->cdm_diff_status == CDM_DIFF_ERROR) {
682 : return CONSDIFF_NOT_FOUND;
683 26 : } else if (ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS) {
684 : return CONSDIFF_IN_PROGRESS;
685 21 : } else if (BUG(ent->cdm_diff_status != CDM_DIFF_PRESENT)) {
686 0 : return CONSDIFF_IN_PROGRESS;
687 : }
688 :
689 21 : if (BUG(ent->entry == NULL)) {
690 0 : return CONSDIFF_NOT_FOUND;
691 : }
692 21 : *entry_out = consensus_cache_entry_handle_get(ent->entry);
693 21 : return (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
694 :
695 : #if 0
696 : // XXXX Remove this. I'm keeping it around for now in case we need to
697 : // XXXX debug issues in the hashtable.
698 : char hex[HEX_DIGEST256_LEN+1];
699 : base16_encode(hex, sizeof(hex), (const char *)digest, digestlen);
700 : const char *flavname = networkstatus_get_flavor_name(flavor);
701 :
702 : smartlist_t *matches = smartlist_new();
703 : consensus_cache_find_all(matches, cdm_cache_get(),
704 : LABEL_FROM_SHA3_DIGEST, hex);
705 : consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
706 : consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
707 :
708 : *entry_out = sort_and_find_most_recent(matches);
709 : consdiff_status_t result =
710 : (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
711 : smartlist_free(matches);
712 :
713 : return result;
714 : #endif /* 0 */
715 : }
716 :
717 : /**
718 : * Perform periodic cleanup tasks on the consensus diff cache. Return
719 : * the number of objects marked for deletion.
720 : */
721 : int
722 18 : consdiffmgr_cleanup(void)
723 : {
724 18 : smartlist_t *objects = smartlist_new();
725 18 : smartlist_t *consensuses = smartlist_new();
726 18 : smartlist_t *diffs = smartlist_new();
727 18 : int n_to_delete = 0;
728 :
729 18 : log_debug(LD_DIRSERV, "Looking for consdiffmgr entries to remove");
730 :
731 : // 1. Delete any consensus or diff or anything whose valid_after is too old.
732 18 : const time_t valid_after_cutoff = approx_time() - get_max_age_to_cache();
733 :
734 18 : consensus_cache_find_all(objects, cdm_cache_get(),
735 : NULL, NULL);
736 212 : SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
737 194 : const char *lv_valid_after =
738 194 : consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
739 194 : if (! lv_valid_after) {
740 5 : log_debug(LD_DIRSERV, "Ignoring entry because it had no %s label",
741 : LABEL_VALID_AFTER);
742 6 : continue;
743 : }
744 189 : time_t valid_after = 0;
745 189 : if (parse_iso_time_nospace(lv_valid_after, &valid_after) < 0) {
746 1 : log_debug(LD_DIRSERV, "Ignoring entry because its %s value (%s) was "
747 : "unparseable", LABEL_VALID_AFTER, escaped(lv_valid_after));
748 1 : continue;
749 : }
750 188 : if (valid_after < valid_after_cutoff) {
751 1 : log_debug(LD_DIRSERV, "Deleting entry because its %s value (%s) was "
752 : "too old", LABEL_VALID_AFTER, lv_valid_after);
753 1 : consensus_cache_entry_mark_for_removal(ent);
754 1 : ++n_to_delete;
755 : }
756 194 : } SMARTLIST_FOREACH_END(ent);
757 :
758 : // 2. Delete all diffs that lead to a consensus whose valid-after is not the
759 : // latest.
760 54 : for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
761 36 : const char *flavname = networkstatus_get_flavor_name(flav);
762 : /* Determine the most recent consensus of this flavor */
763 36 : consensus_cache_find_all(consensuses, cdm_cache_get(),
764 : LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
765 36 : consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
766 36 : consensus_cache_entry_t *most_recent =
767 36 : sort_and_find_most_recent(consensuses);
768 36 : if (most_recent == NULL)
769 18 : continue;
770 18 : const char *most_recent_sha3 =
771 18 : consensus_cache_entry_get_value(most_recent,
772 : LABEL_SHA3_DIGEST_UNCOMPRESSED);
773 18 : if (BUG(most_recent_sha3 == NULL))
774 : continue; // LCOV_EXCL_LINE
775 :
776 : /* consider all such-flavored diffs, and look to see if they match. */
777 18 : consensus_cache_find_all(diffs, cdm_cache_get(),
778 : LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
779 18 : consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
780 90 : SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
781 72 : const char *this_diff_target_sha3 =
782 72 : consensus_cache_entry_get_value(diff, LABEL_TARGET_SHA3_DIGEST);
783 72 : if (!this_diff_target_sha3)
784 0 : continue;
785 72 : if (strcmp(this_diff_target_sha3, most_recent_sha3)) {
786 8 : consensus_cache_entry_mark_for_removal(diff);
787 8 : ++n_to_delete;
788 : }
789 72 : } SMARTLIST_FOREACH_END(diff);
790 18 : smartlist_clear(consensuses);
791 18 : smartlist_clear(diffs);
792 : }
793 :
794 : // 3. Delete all consensuses except the most recent that are compressed with
795 : // an un-preferred method.
796 54 : for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
797 36 : const char *flavname = networkstatus_get_flavor_name(flav);
798 : /* Determine the most recent consensus of this flavor */
799 36 : consensus_cache_find_all(consensuses, cdm_cache_get(),
800 : LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
801 36 : consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
802 36 : consensus_cache_entry_t *most_recent =
803 36 : sort_and_find_most_recent(consensuses);
804 36 : if (most_recent == NULL)
805 18 : continue;
806 18 : const char *most_recent_sha3_uncompressed =
807 18 : consensus_cache_entry_get_value(most_recent,
808 : LABEL_SHA3_DIGEST_UNCOMPRESSED);
809 18 : const char *retain_methodname = compression_method_get_name(
810 : RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
811 :
812 18 : if (BUG(most_recent_sha3_uncompressed == NULL))
813 0 : continue;
814 133 : SMARTLIST_FOREACH_BEGIN(consensuses, consensus_cache_entry_t *, ent) {
815 115 : const char *lv_sha3_uncompressed =
816 115 : consensus_cache_entry_get_value(ent, LABEL_SHA3_DIGEST_UNCOMPRESSED);
817 115 : if (BUG(! lv_sha3_uncompressed))
818 0 : continue;
819 115 : if (!strcmp(lv_sha3_uncompressed, most_recent_sha3_uncompressed))
820 54 : continue; // This _is_ the most recent.
821 61 : const char *lv_methodname =
822 61 : consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
823 61 : if (! lv_methodname || strcmp(lv_methodname, retain_methodname)) {
824 26 : consensus_cache_entry_mark_for_removal(ent);
825 26 : ++n_to_delete;
826 : }
827 115 : } SMARTLIST_FOREACH_END(ent);
828 : }
829 :
830 18 : smartlist_free(objects);
831 18 : smartlist_free(consensuses);
832 18 : smartlist_free(diffs);
833 :
834 : // Actually remove files, if they're not used.
835 18 : consensus_cache_delete_pending(cdm_cache_get(), 0);
836 18 : return n_to_delete;
837 : }
838 :
839 : /**
840 : * Initialize the consensus diff manager and its cache, and configure
841 : * its parameters based on the latest torrc and networkstatus parameters.
842 : */
843 : void
844 15 : consdiffmgr_configure(const consdiff_cfg_t *cfg)
845 : {
846 15 : if (cfg)
847 11 : memcpy(&consdiff_cfg, cfg, sizeof(consdiff_cfg));
848 :
849 15 : (void) cdm_cache_get();
850 15 : }
851 :
852 : /**
853 : * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
854 : * operations that the consensus diff manager will need.
855 : */
856 : int
857 0 : consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem_t **cfg)
858 : {
859 0 : return consensus_cache_register_with_sandbox(cdm_cache_get(), cfg);
860 : }
861 :
862 : /**
863 : * Scan the consensus diff manager's cache for any grossly malformed entries,
864 : * and mark them as deletable. Return 0 if no problems were found; 1
865 : * if problems were found and fixed.
866 : */
867 : int
868 6 : consdiffmgr_validate(void)
869 : {
870 : /* Right now, we only check for entries that have bad sha3 values */
871 6 : int problems = 0;
872 :
873 6 : smartlist_t *objects = smartlist_new();
874 6 : consensus_cache_find_all(objects, cdm_cache_get(),
875 : NULL, NULL);
876 16 : SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, obj) {
877 10 : uint8_t sha3_expected[DIGEST256_LEN];
878 10 : uint8_t sha3_received[DIGEST256_LEN];
879 10 : int r = cdm_entry_get_sha3_value(sha3_expected, obj, LABEL_SHA3_DIGEST);
880 10 : if (r == -1) {
881 : /* digest isn't there; that's allowed */
882 3 : continue;
883 9 : } else if (r == -2) {
884 : /* digest is malformed; that's not allowed */
885 1 : problems = 1;
886 1 : consensus_cache_entry_mark_for_removal(obj);
887 1 : continue;
888 : }
889 8 : const uint8_t *body;
890 8 : size_t bodylen;
891 8 : consensus_cache_entry_incref(obj);
892 8 : r = consensus_cache_entry_get_body(obj, &body, &bodylen);
893 8 : if (r == 0) {
894 8 : crypto_digest256((char *)sha3_received, (const char *)body, bodylen,
895 : DIGEST_SHA3_256);
896 : }
897 8 : consensus_cache_entry_decref(obj);
898 8 : if (r < 0)
899 0 : continue;
900 :
901 : // Deconfuse coverity about the possibility of sha3_received being
902 : // uninitialized
903 8 : tor_assert(r <= 0);
904 :
905 8 : if (fast_memneq(sha3_received, sha3_expected, DIGEST256_LEN)) {
906 1 : problems = 1;
907 1 : consensus_cache_entry_mark_for_removal(obj);
908 1 : continue;
909 : }
910 :
911 10 : } SMARTLIST_FOREACH_END(obj);
912 6 : smartlist_free(objects);
913 6 : return problems;
914 : }
915 :
916 : /**
917 : * Helper: build new diffs of <b>flavor</b> as needed
918 : */
919 : static void
920 26 : consdiffmgr_rescan_flavor_(consensus_flavor_t flavor)
921 : {
922 26 : smartlist_t *matches = NULL;
923 26 : smartlist_t *diffs = NULL;
924 26 : smartlist_t *compute_diffs_from = NULL;
925 26 : strmap_t *have_diff_from = NULL;
926 :
927 : // look for the most recent consensus, and for all previous in-range
928 : // consensuses. Do they all have diffs to it?
929 26 : const char *flavname = networkstatus_get_flavor_name(flavor);
930 :
931 : // 1. find the most recent consensus, and the ones that we might want
932 : // to diff to it.
933 26 : const char *methodname = compression_method_get_name(
934 : RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
935 :
936 26 : matches = smartlist_new();
937 26 : consensus_cache_find_all(matches, cdm_cache_get(),
938 : LABEL_FLAVOR, flavname);
939 26 : consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
940 26 : consensus_cache_filter_list(matches, LABEL_COMPRESSION_TYPE, methodname);
941 26 : consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
942 26 : if (!most_recent) {
943 10 : log_info(LD_DIRSERV, "No 'most recent' %s consensus found; "
944 : "not making diffs", flavname);
945 10 : goto done;
946 : }
947 16 : tor_assert(smartlist_len(matches));
948 16 : smartlist_del(matches, smartlist_len(matches) - 1);
949 :
950 16 : const char *most_recent_valid_after =
951 16 : consensus_cache_entry_get_value(most_recent, LABEL_VALID_AFTER);
952 16 : if (BUG(most_recent_valid_after == NULL))
953 : goto done; //LCOV_EXCL_LINE
954 16 : uint8_t most_recent_sha3[DIGEST256_LEN];
955 16 : if (BUG(cdm_entry_get_sha3_value(most_recent_sha3, most_recent,
956 : LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
957 : goto done; //LCOV_EXCL_LINE
958 :
959 : // 2. Find all the relevant diffs _to_ this consensus. These are ones
960 : // that we don't need to compute.
961 16 : diffs = smartlist_new();
962 16 : consensus_cache_find_all(diffs, cdm_cache_get(),
963 : LABEL_VALID_AFTER, most_recent_valid_after);
964 16 : consensus_cache_filter_list(diffs, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
965 16 : consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
966 16 : have_diff_from = strmap_new();
967 72 : SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
968 56 : const char *va = consensus_cache_entry_get_value(diff,
969 : LABEL_FROM_VALID_AFTER);
970 56 : if (BUG(va == NULL))
971 : continue; // LCOV_EXCL_LINE
972 56 : strmap_set(have_diff_from, va, diff);
973 56 : } SMARTLIST_FOREACH_END(diff);
974 :
975 : // 3. See which consensuses in 'matches' don't have diffs yet.
976 16 : smartlist_reverse(matches); // from newest to oldest.
977 16 : compute_diffs_from = smartlist_new();
978 46 : SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
979 30 : const char *va = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
980 30 : if (BUG(va == NULL))
981 : continue; // LCOV_EXCL_LINE
982 30 : if (strmap_get(have_diff_from, va) != NULL)
983 14 : continue; /* we already have this one. */
984 16 : smartlist_add(compute_diffs_from, ent);
985 : /* Since we are not going to serve this as the most recent consensus
986 : * any more, we should stop keeping it mmap'd when it's not in use.
987 : */
988 16 : consensus_cache_entry_mark_for_aggressive_release(ent);
989 30 : } SMARTLIST_FOREACH_END(ent);
990 :
991 16 : log_info(LD_DIRSERV,
992 : "The most recent %s consensus is valid-after %s. We have diffs to "
993 : "this consensus for %d/%d older %s consensuses. Generating diffs "
994 : "for the other %d.",
995 : flavname,
996 : most_recent_valid_after,
997 : smartlist_len(matches) - smartlist_len(compute_diffs_from),
998 : smartlist_len(matches),
999 : flavname,
1000 : smartlist_len(compute_diffs_from));
1001 :
1002 : // 4. Update the hashtable; remove entries in this flavor to other
1003 : // target consensuses.
1004 16 : cdm_diff_ht_purge(flavor, most_recent_sha3);
1005 :
1006 : // 5. Actually launch the requests.
1007 32 : SMARTLIST_FOREACH_BEGIN(compute_diffs_from, consensus_cache_entry_t *, c) {
1008 16 : if (BUG(c == most_recent))
1009 : continue; // LCOV_EXCL_LINE
1010 :
1011 16 : uint8_t this_sha3[DIGEST256_LEN];
1012 16 : if (cdm_entry_get_sha3_value(this_sha3, c,
1013 : LABEL_SHA3_DIGEST_AS_SIGNED)<0) {
1014 : // Not actually a bug, since we might be running with a directory
1015 : // with stale files from before the #22143 fixes.
1016 0 : continue;
1017 : }
1018 16 : if (cdm_diff_ht_check_and_note_pending(flavor,
1019 : this_sha3, most_recent_sha3)) {
1020 : // This is already pending, or we encountered an error.
1021 1 : continue;
1022 : }
1023 15 : consensus_diff_queue_diff_work(c, most_recent);
1024 16 : } SMARTLIST_FOREACH_END(c);
1025 :
1026 16 : done:
1027 26 : smartlist_free(matches);
1028 26 : smartlist_free(diffs);
1029 26 : smartlist_free(compute_diffs_from);
1030 26 : strmap_free(have_diff_from, NULL);
1031 26 : }
1032 :
1033 : /**
1034 : * Scan the cache for the latest consensuses and add their handles to
1035 : * latest_consensus
1036 : */
1037 : static void
1038 7 : consdiffmgr_consensus_load(void)
1039 : {
1040 7 : smartlist_t *matches = smartlist_new();
1041 21 : for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
1042 14 : const char *flavname = networkstatus_get_flavor_name(flav);
1043 14 : smartlist_clear(matches);
1044 14 : consensus_cache_find_all(matches, cdm_cache_get(),
1045 : LABEL_FLAVOR, flavname);
1046 14 : consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
1047 14 : consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
1048 14 : if (! most_recent)
1049 8 : continue; // no consensuses.
1050 6 : const char *most_recent_sha3 =
1051 6 : consensus_cache_entry_get_value(most_recent,
1052 : LABEL_SHA3_DIGEST_UNCOMPRESSED);
1053 6 : if (BUG(most_recent_sha3 == NULL))
1054 : continue; // LCOV_EXCL_LINE
1055 6 : consensus_cache_filter_list(matches, LABEL_SHA3_DIGEST_UNCOMPRESSED,
1056 : most_recent_sha3);
1057 :
1058 : // Everything that remains matches the most recent consensus of this
1059 : // flavor.
1060 24 : SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
1061 18 : const char *lv_compression =
1062 18 : consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
1063 18 : compress_method_t method =
1064 18 : compression_method_get_by_name(lv_compression);
1065 18 : int pos = consensus_compression_method_pos(method);
1066 18 : if (pos < 0)
1067 0 : continue;
1068 18 : consensus_cache_entry_handle_free(latest_consensus[flav][pos]);
1069 18 : latest_consensus[flav][pos] = consensus_cache_entry_handle_new(ent);
1070 18 : } SMARTLIST_FOREACH_END(ent);
1071 : }
1072 7 : smartlist_free(matches);
1073 7 : }
1074 :
1075 : /**
1076 : * Scan the cache for diffs, and add them to the hashtable.
1077 : */
1078 : static void
1079 7 : consdiffmgr_diffs_load(void)
1080 : {
1081 7 : smartlist_t *diffs = smartlist_new();
1082 7 : consensus_cache_find_all(diffs, cdm_cache_get(),
1083 : LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
1084 31 : SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
1085 24 : const char *lv_flavor =
1086 24 : consensus_cache_entry_get_value(diff, LABEL_FLAVOR);
1087 24 : if (!lv_flavor)
1088 0 : continue;
1089 24 : int flavor = networkstatus_parse_flavor_name(lv_flavor);
1090 24 : if (flavor < 0)
1091 0 : continue;
1092 24 : const char *lv_compression =
1093 24 : consensus_cache_entry_get_value(diff, LABEL_COMPRESSION_TYPE);
1094 24 : compress_method_t method = NO_METHOD;
1095 24 : if (lv_compression) {
1096 18 : method = compression_method_get_by_name(lv_compression);
1097 18 : if (method == UNKNOWN_METHOD) {
1098 0 : continue;
1099 : }
1100 : }
1101 :
1102 24 : uint8_t from_sha3[DIGEST256_LEN];
1103 24 : uint8_t to_sha3[DIGEST256_LEN];
1104 24 : if (cdm_entry_get_sha3_value(from_sha3, diff, LABEL_FROM_SHA3_DIGEST)<0)
1105 0 : continue;
1106 24 : if (cdm_entry_get_sha3_value(to_sha3, diff, LABEL_TARGET_SHA3_DIGEST)<0)
1107 0 : continue;
1108 :
1109 24 : cdm_diff_ht_set_status(flavor, from_sha3, to_sha3,
1110 : method,
1111 : CDM_DIFF_PRESENT,
1112 : consensus_cache_entry_handle_new(diff));
1113 24 : } SMARTLIST_FOREACH_END(diff);
1114 7 : smartlist_free(diffs);
1115 7 : }
1116 :
1117 : /**
1118 : * Build new diffs as needed.
1119 : */
1120 : void
1121 14 : consdiffmgr_rescan(void)
1122 : {
1123 14 : if (cdm_cache_dirty == 0)
1124 : return;
1125 :
1126 : // Clean up here to make room for new diffs, and to ensure that older
1127 : // consensuses do not have any entries.
1128 13 : consdiffmgr_cleanup();
1129 :
1130 13 : if (cdm_cache_loaded == 0) {
1131 7 : consdiffmgr_diffs_load();
1132 7 : consdiffmgr_consensus_load();
1133 7 : cdm_cache_loaded = 1;
1134 : }
1135 :
1136 39 : for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
1137 26 : consdiffmgr_rescan_flavor_((consensus_flavor_t) flav);
1138 : }
1139 :
1140 13 : cdm_cache_dirty = 0;
1141 : }
1142 :
1143 : /** Callback wrapper for consdiffmgr_rescan */
1144 : static void
1145 0 : consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg)
1146 : {
1147 0 : (void)ev;
1148 0 : (void)arg;
1149 0 : consdiffmgr_rescan();
1150 0 : }
1151 :
1152 : /** Mark the cache as dirty, and schedule a rescan event. */
1153 : static void
1154 49 : mark_cdm_cache_dirty(void)
1155 : {
1156 49 : cdm_cache_dirty = 1;
1157 49 : tor_assert(consdiffmgr_rescan_ev);
1158 49 : mainloop_event_activate(consdiffmgr_rescan_ev);
1159 49 : }
1160 :
1161 : /**
1162 : * Helper: compare two files by their from-valid-after and valid-after labels,
1163 : * trying to sort in ascending order by from-valid-after (when present) and
1164 : * valid-after (when not). Place everything that has neither label first in
1165 : * the list.
1166 : */
1167 : static int
1168 0 : compare_by_staleness_(const void **a, const void **b)
1169 : {
1170 0 : const consensus_cache_entry_t *e1 = *a;
1171 0 : const consensus_cache_entry_t *e2 = *b;
1172 0 : const char *va1, *fva1, *va2, *fva2;
1173 0 : va1 = consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER);
1174 0 : va2 = consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER);
1175 0 : fva1 = consensus_cache_entry_get_value(e1, LABEL_FROM_VALID_AFTER);
1176 0 : fva2 = consensus_cache_entry_get_value(e2, LABEL_FROM_VALID_AFTER);
1177 :
1178 0 : if (fva1)
1179 0 : va1 = fva1;
1180 0 : if (fva2)
1181 0 : va2 = fva2;
1182 :
1183 : /* See note about iso-encoded values in compare_by_valid_after_. Also note
1184 : * that missing dates will get placed first. */
1185 0 : return strcmp_opt(va1, va2);
1186 : }
1187 :
1188 : /** If there are not enough unused filenames to store <b>n</b> files, then
1189 : * delete old consensuses until there are. (We have to keep track of the
1190 : * number of filenames because of the way that the seccomp2 cache works.)
1191 : *
1192 : * Return 0 on success, -1 on failure.
1193 : **/
1194 : static int
1195 40 : consdiffmgr_ensure_space_for_files(int n)
1196 : {
1197 40 : consensus_cache_t *cache = cdm_cache_get();
1198 40 : if (consensus_cache_get_n_filenames_available(cache) >= n) {
1199 : // there are already enough unused filenames.
1200 : return 0;
1201 : }
1202 : // Try a cheap deletion of stuff that's waiting to get deleted.
1203 0 : consensus_cache_delete_pending(cache, 0);
1204 0 : if (consensus_cache_get_n_filenames_available(cache) >= n) {
1205 : // okay, _that_ made enough filenames available.
1206 : return 0;
1207 : }
1208 : // Let's get more assertive: clean out unused stuff, and force-remove
1209 : // the files that we can.
1210 0 : consdiffmgr_cleanup();
1211 0 : consensus_cache_delete_pending(cache, 1);
1212 0 : const int n_to_remove = n - consensus_cache_get_n_filenames_available(cache);
1213 0 : if (n_to_remove <= 0) {
1214 : // okay, finally!
1215 : return 0;
1216 : }
1217 :
1218 : // At this point, we're going to have to throw out objects that will be
1219 : // missed. Too bad!
1220 0 : smartlist_t *objects = smartlist_new();
1221 0 : consensus_cache_find_all(objects, cache, NULL, NULL);
1222 0 : smartlist_sort(objects, compare_by_staleness_);
1223 0 : int n_marked = 0;
1224 0 : SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
1225 0 : consensus_cache_entry_mark_for_removal(ent);
1226 0 : if (++n_marked >= n_to_remove)
1227 : break;
1228 0 : } SMARTLIST_FOREACH_END(ent);
1229 0 : smartlist_free(objects);
1230 :
1231 0 : consensus_cache_delete_pending(cache, 1);
1232 :
1233 0 : if (consensus_cache_may_overallocate(cache)) {
1234 : /* If we're allowed to throw extra files into the cache, let's do so
1235 : * rather getting upset.
1236 : */
1237 : return 0;
1238 : }
1239 :
1240 0 : if (BUG(n_marked < n_to_remove))
1241 0 : return -1;
1242 : else
1243 : return 0;
1244 : }
1245 :
1246 : /**
1247 : * Set consensus cache flags on the objects in this consdiffmgr.
1248 : */
1249 : static void
1250 21 : consdiffmgr_set_cache_flags(void)
1251 : {
1252 : /* Right now, we just mark the consensus objects for aggressive release,
1253 : * so that they get mmapped for as little time as possible. */
1254 21 : smartlist_t *objects = smartlist_new();
1255 21 : consensus_cache_find_all(objects, cdm_cache_get(), LABEL_DOCTYPE,
1256 : DOCTYPE_CONSENSUS);
1257 36 : SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
1258 15 : consensus_cache_entry_mark_for_aggressive_release(ent);
1259 15 : } SMARTLIST_FOREACH_END(ent);
1260 21 : smartlist_free(objects);
1261 21 : }
1262 :
1263 : /**
1264 : * Called before shutdown: drop all storage held by the consdiffmgr.c module.
1265 : */
1266 : void
1267 248 : consdiffmgr_free_all(void)
1268 : {
1269 248 : cdm_diff_t **diff, **next;
1270 324 : for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
1271 76 : cdm_diff_t *this = *diff;
1272 76 : next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
1273 76 : cdm_diff_free(this);
1274 : }
1275 : int i;
1276 : unsigned j;
1277 744 : for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
1278 1984 : for (j = 0; j < n_consensus_compression_methods(); ++j) {
1279 1488 : consensus_cache_entry_handle_free(latest_consensus[i][j]);
1280 : }
1281 : }
1282 248 : memset(latest_consensus, 0, sizeof(latest_consensus));
1283 248 : consensus_cache_free(cons_diff_cache);
1284 248 : cons_diff_cache = NULL;
1285 248 : mainloop_event_free(consdiffmgr_rescan_ev);
1286 248 : }
1287 :
1288 : /* =====
1289 : Thread workers
1290 : =====*/
1291 :
1292 : typedef struct compressed_result_t {
1293 : config_line_t *labels;
1294 : /**
1295 : * Output: Body of the diff, as compressed.
1296 : */
1297 : uint8_t *body;
1298 : /**
1299 : * Output: length of body_out
1300 : */
1301 : size_t bodylen;
1302 : } compressed_result_t;
1303 :
1304 : /**
1305 : * Compress the bytestring <b>input</b> of length <b>len</b> using the
1306 : * <b>n_methods</b> compression methods listed in the array <b>methods</b>.
1307 : *
1308 : * For each successful compression, set the fields in the <b>results_out</b>
1309 : * array in the position corresponding to the compression method. Use
1310 : * <b>labels_in</b> as a basis for the labels of the result.
1311 : *
1312 : * Return 0 if all compression succeeded; -1 if any failed.
1313 : */
1314 : static int
1315 39 : compress_multiple(compressed_result_t *results_out, int n_methods,
1316 : const compress_method_t *methods,
1317 : const uint8_t *input, size_t len,
1318 : const config_line_t *labels_in)
1319 : {
1320 39 : int rv = 0;
1321 39 : int i;
1322 156 : for (i = 0; i < n_methods; ++i) {
1323 117 : compress_method_t method = methods[i];
1324 117 : const char *methodname = compression_method_get_name(method);
1325 117 : char *result;
1326 117 : size_t sz;
1327 117 : if (0 == tor_compress(&result, &sz, (const char*)input, len, method)) {
1328 117 : results_out[i].body = (uint8_t*)result;
1329 117 : results_out[i].bodylen = sz;
1330 117 : results_out[i].labels = config_lines_dup(labels_in);
1331 117 : cdm_labels_prepend_sha3(&results_out[i].labels, LABEL_SHA3_DIGEST,
1332 117 : results_out[i].body,
1333 : results_out[i].bodylen);
1334 117 : config_line_prepend(&results_out[i].labels,
1335 : LABEL_COMPRESSION_TYPE,
1336 : methodname);
1337 : } else {
1338 : rv = -1;
1339 : }
1340 : }
1341 39 : return rv;
1342 : }
1343 :
1344 : /**
1345 : * Given an array of <b>n</b> compressed_result_t in <b>results</b>,
1346 : * as produced by compress_multiple, store them all into the
1347 : * consdiffmgr, and store handles to them in the <b>handles_out</b>
1348 : * array.
1349 : *
1350 : * Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none
1351 : * was stored.
1352 : */
1353 : static cdm_diff_status_t
1354 40 : store_multiple(consensus_cache_entry_handle_t **handles_out,
1355 : int n,
1356 : const compress_method_t *methods,
1357 : const compressed_result_t *results,
1358 : const char *description)
1359 : {
1360 40 : cdm_diff_status_t status = CDM_DIFF_ERROR;
1361 40 : consdiffmgr_ensure_space_for_files(n);
1362 :
1363 40 : int i;
1364 212 : for (i = 0; i < n; ++i) {
1365 132 : compress_method_t method = methods[i];
1366 132 : uint8_t *body_out = results[i].body;
1367 132 : size_t bodylen_out = results[i].bodylen;
1368 132 : config_line_t *labels = results[i].labels;
1369 132 : const char *methodname = compression_method_get_name(method);
1370 132 : if (body_out && bodylen_out && labels) {
1371 : /* Success! Store the results */
1372 128 : log_info(LD_DIRSERV, "Adding %s, compressed with %s",
1373 : description, methodname);
1374 :
1375 128 : consensus_cache_entry_t *ent =
1376 128 : consensus_cache_add(cdm_cache_get(),
1377 : labels,
1378 : body_out,
1379 : bodylen_out);
1380 128 : if (ent == NULL) {
1381 0 : static ratelim_t cant_store_ratelim = RATELIM_INIT(5*60);
1382 0 : log_fn_ratelim(&cant_store_ratelim, LOG_WARN, LD_FS,
1383 : "Unable to store object %s compressed with %s.",
1384 : description, methodname);
1385 0 : continue;
1386 : }
1387 :
1388 128 : status = CDM_DIFF_PRESENT;
1389 128 : handles_out[i] = consensus_cache_entry_handle_new(ent);
1390 128 : consensus_cache_entry_decref(ent);
1391 : }
1392 : }
1393 40 : return status;
1394 : }
1395 :
1396 : /**
1397 : * An object passed to a worker thread that will try to produce a consensus
1398 : * diff.
1399 : */
1400 : typedef struct consensus_diff_worker_job_t {
1401 : /**
1402 : * Input: The consensus to compute the diff from. Holds a reference to the
1403 : * cache entry, which must not be released until the job is passed back to
1404 : * the main thread. The body must be mapped into memory in the main thread.
1405 : */
1406 : consensus_cache_entry_t *diff_from;
1407 : /**
1408 : * Input: The consensus to compute the diff to. Holds a reference to the
1409 : * cache entry, which must not be released until the job is passed back to
1410 : * the main thread. The body must be mapped into memory in the main thread.
1411 : */
1412 : consensus_cache_entry_t *diff_to;
1413 :
1414 : /** Output: labels and bodies */
1415 : compressed_result_t out[ARRAY_LENGTH(compress_diffs_with)];
1416 : } consensus_diff_worker_job_t;
1417 :
1418 : /** Given a consensus_cache_entry_t, check whether it has a label claiming
1419 : * that it was compressed. If so, uncompress its contents into *<b>out</b> and
1420 : * set <b>outlen</b> to hold their size, and set *<b>owned_out</b> to a pointer
1421 : * that the caller will need to free. If not, just set *<b>out</b> and
1422 : * <b>outlen</b> to its extent in memory. Return 0 on success, -1 on failure.
1423 : **/
1424 : STATIC int
1425 42 : uncompress_or_set_ptr(const char **out, size_t *outlen,
1426 : char **owned_out,
1427 : consensus_cache_entry_t *ent)
1428 : {
1429 42 : const uint8_t *body;
1430 42 : size_t bodylen;
1431 :
1432 42 : *owned_out = NULL;
1433 :
1434 42 : if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0)
1435 : return -1;
1436 :
1437 42 : const char *lv_compression =
1438 42 : consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
1439 42 : compress_method_t method = NO_METHOD;
1440 :
1441 42 : if (lv_compression)
1442 25 : method = compression_method_get_by_name(lv_compression);
1443 :
1444 25 : int rv;
1445 25 : if (method == NO_METHOD) {
1446 17 : *out = (const char *)body;
1447 17 : *outlen = bodylen;
1448 17 : rv = 0;
1449 : } else {
1450 25 : rv = tor_uncompress(owned_out, outlen, (const char *)body, bodylen,
1451 : method, 1, LOG_WARN);
1452 25 : *out = *owned_out;
1453 : }
1454 : return rv;
1455 : }
1456 :
1457 : /**
1458 : * Worker function. This function runs inside a worker thread and receives
1459 : * a consensus_diff_worker_job_t as its input.
1460 : */
1461 : static workqueue_reply_t
1462 12 : consensus_diff_worker_threadfn(void *state_, void *work_)
1463 : {
1464 12 : (void)state_;
1465 12 : consensus_diff_worker_job_t *job = work_;
1466 12 : const uint8_t *diff_from, *diff_to;
1467 12 : size_t len_from, len_to;
1468 12 : int r;
1469 : /* We need to have the body already mapped into RAM here.
1470 : */
1471 12 : r = consensus_cache_entry_get_body(job->diff_from, &diff_from, &len_from);
1472 12 : if (BUG(r < 0))
1473 : return WQ_RPL_REPLY; // LCOV_EXCL_LINE
1474 12 : r = consensus_cache_entry_get_body(job->diff_to, &diff_to, &len_to);
1475 12 : if (BUG(r < 0))
1476 : return WQ_RPL_REPLY; // LCOV_EXCL_LINE
1477 :
1478 12 : const char *lv_to_valid_after =
1479 12 : consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER);
1480 12 : const char *lv_to_fresh_until =
1481 12 : consensus_cache_entry_get_value(job->diff_to, LABEL_FRESH_UNTIL);
1482 12 : const char *lv_to_valid_until =
1483 12 : consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_UNTIL);
1484 12 : const char *lv_to_signatories =
1485 12 : consensus_cache_entry_get_value(job->diff_to, LABEL_SIGNATORIES);
1486 12 : const char *lv_from_valid_after =
1487 12 : consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER);
1488 12 : const char *lv_from_digest =
1489 12 : consensus_cache_entry_get_value(job->diff_from,
1490 : LABEL_SHA3_DIGEST_AS_SIGNED);
1491 12 : const char *lv_from_flavor =
1492 12 : consensus_cache_entry_get_value(job->diff_from, LABEL_FLAVOR);
1493 12 : const char *lv_to_flavor =
1494 12 : consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
1495 12 : const char *lv_to_digest =
1496 12 : consensus_cache_entry_get_value(job->diff_to,
1497 : LABEL_SHA3_DIGEST_UNCOMPRESSED);
1498 :
1499 12 : if (! lv_from_digest) {
1500 : /* This isn't a bug right now, since it can happen if you're migrating
1501 : * from an older version of master to a newer one. The older ones didn't
1502 : * annotate their stored consensus objects with sha3-digest-as-signed.
1503 : */
1504 : return WQ_RPL_REPLY; // LCOV_EXCL_LINE
1505 : }
1506 :
1507 : /* All these values are mandatory on the input */
1508 12 : if (BUG(!lv_to_valid_after) ||
1509 12 : BUG(!lv_from_valid_after) ||
1510 12 : BUG(!lv_from_flavor) ||
1511 12 : BUG(!lv_to_flavor)) {
1512 : return WQ_RPL_REPLY; // LCOV_EXCL_LINE
1513 : }
1514 : /* The flavors need to match */
1515 12 : if (BUG(strcmp(lv_from_flavor, lv_to_flavor))) {
1516 : return WQ_RPL_REPLY; // LCOV_EXCL_LINE
1517 : }
1518 :
1519 12 : char *consensus_diff;
1520 : {
1521 12 : const char *diff_from_nt = NULL, *diff_to_nt = NULL;
1522 12 : char *owned1 = NULL, *owned2 = NULL;
1523 12 : size_t diff_from_nt_len, diff_to_nt_len;
1524 :
1525 12 : if (uncompress_or_set_ptr(&diff_from_nt, &diff_from_nt_len, &owned1,
1526 12 : job->diff_from) < 0) {
1527 0 : return WQ_RPL_REPLY;
1528 : }
1529 12 : if (uncompress_or_set_ptr(&diff_to_nt, &diff_to_nt_len, &owned2,
1530 12 : job->diff_to) < 0) {
1531 0 : tor_free(owned1);
1532 0 : return WQ_RPL_REPLY;
1533 : }
1534 12 : tor_assert(diff_from_nt);
1535 12 : tor_assert(diff_to_nt);
1536 :
1537 : // XXXX ugh; this is going to calculate the SHA3 of both its
1538 : // XXXX inputs again, even though we already have that. Maybe it's time
1539 : // XXXX to change the API here?
1540 12 : consensus_diff = consensus_diff_generate(diff_from_nt,
1541 : diff_from_nt_len,
1542 : diff_to_nt,
1543 : diff_to_nt_len);
1544 12 : tor_free(owned1);
1545 12 : tor_free(owned2);
1546 : }
1547 12 : if (!consensus_diff) {
1548 : /* Couldn't generate consensus; we'll leave the reply blank. */
1549 : return WQ_RPL_REPLY;
1550 : }
1551 :
1552 : /* Compress the results and send the reply */
1553 11 : tor_assert(compress_diffs_with[0] == NO_METHOD);
1554 11 : size_t difflen = strlen(consensus_diff);
1555 11 : job->out[0].body = (uint8_t *) consensus_diff;
1556 11 : job->out[0].bodylen = difflen;
1557 :
1558 11 : config_line_t *common_labels = NULL;
1559 11 : if (lv_to_valid_until)
1560 11 : config_line_prepend(&common_labels, LABEL_VALID_UNTIL, lv_to_valid_until);
1561 11 : if (lv_to_fresh_until)
1562 11 : config_line_prepend(&common_labels, LABEL_FRESH_UNTIL, lv_to_fresh_until);
1563 11 : if (lv_to_signatories)
1564 0 : config_line_prepend(&common_labels, LABEL_SIGNATORIES, lv_to_signatories);
1565 11 : cdm_labels_prepend_sha3(&common_labels,
1566 : LABEL_SHA3_DIGEST_UNCOMPRESSED,
1567 11 : job->out[0].body,
1568 : job->out[0].bodylen);
1569 11 : config_line_prepend(&common_labels, LABEL_FROM_VALID_AFTER,
1570 : lv_from_valid_after);
1571 11 : config_line_prepend(&common_labels, LABEL_VALID_AFTER,
1572 : lv_to_valid_after);
1573 11 : config_line_prepend(&common_labels, LABEL_FLAVOR, lv_from_flavor);
1574 11 : config_line_prepend(&common_labels, LABEL_FROM_SHA3_DIGEST,
1575 : lv_from_digest);
1576 11 : config_line_prepend(&common_labels, LABEL_TARGET_SHA3_DIGEST,
1577 : lv_to_digest);
1578 11 : config_line_prepend(&common_labels, LABEL_DOCTYPE,
1579 : DOCTYPE_CONSENSUS_DIFF);
1580 :
1581 11 : job->out[0].labels = config_lines_dup(common_labels);
1582 11 : cdm_labels_prepend_sha3(&job->out[0].labels,
1583 : LABEL_SHA3_DIGEST,
1584 11 : job->out[0].body,
1585 : job->out[0].bodylen);
1586 :
1587 11 : compress_multiple(job->out+1,
1588 11 : n_diff_compression_methods()-1,
1589 : compress_diffs_with+1,
1590 : (const uint8_t*)consensus_diff, difflen, common_labels);
1591 :
1592 11 : config_free_lines(common_labels);
1593 11 : return WQ_RPL_REPLY;
1594 : }
1595 :
1596 : #define consensus_diff_worker_job_free(job) \
1597 : FREE_AND_NULL(consensus_diff_worker_job_t, \
1598 : consensus_diff_worker_job_free_, (job))
1599 :
1600 : /**
1601 : * Helper: release all storage held in <b>job</b>.
1602 : */
1603 : static void
1604 12 : consensus_diff_worker_job_free_(consensus_diff_worker_job_t *job)
1605 : {
1606 12 : if (!job)
1607 : return;
1608 : unsigned u;
1609 60 : for (u = 0; u < n_diff_compression_methods(); ++u) {
1610 48 : config_free_lines(job->out[u].labels);
1611 48 : tor_free(job->out[u].body);
1612 : }
1613 12 : consensus_cache_entry_decref(job->diff_from);
1614 12 : consensus_cache_entry_decref(job->diff_to);
1615 12 : tor_free(job);
1616 : }
1617 :
1618 : /**
1619 : * Worker function: This function runs in the main thread, and receives
1620 : * a consensus_diff_worker_job_t that the worker thread has already
1621 : * processed.
1622 : */
1623 : static void
1624 12 : consensus_diff_worker_replyfn(void *work_)
1625 : {
1626 12 : tor_assert(in_main_thread());
1627 12 : tor_assert(work_);
1628 :
1629 12 : consensus_diff_worker_job_t *job = work_;
1630 :
1631 12 : const char *lv_from_digest =
1632 12 : consensus_cache_entry_get_value(job->diff_from,
1633 : LABEL_SHA3_DIGEST_AS_SIGNED);
1634 12 : const char *lv_to_digest =
1635 12 : consensus_cache_entry_get_value(job->diff_to,
1636 : LABEL_SHA3_DIGEST_UNCOMPRESSED);
1637 12 : const char *lv_flavor =
1638 12 : consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
1639 12 : if (BUG(lv_from_digest == NULL))
1640 : lv_from_digest = "???"; // LCOV_EXCL_LINE
1641 12 : if (BUG(lv_to_digest == NULL))
1642 : lv_to_digest = "???"; // LCOV_EXCL_LINE
1643 :
1644 12 : uint8_t from_sha3[DIGEST256_LEN];
1645 12 : uint8_t to_sha3[DIGEST256_LEN];
1646 12 : int flav = -1;
1647 12 : int cache = 1;
1648 12 : if (BUG(cdm_entry_get_sha3_value(from_sha3, job->diff_from,
1649 : LABEL_SHA3_DIGEST_AS_SIGNED) < 0))
1650 : cache = 0;
1651 12 : if (BUG(cdm_entry_get_sha3_value(to_sha3, job->diff_to,
1652 : LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
1653 : cache = 0;
1654 12 : if (BUG(lv_flavor == NULL)) {
1655 : cache = 0;
1656 12 : } else if ((flav = networkstatus_parse_flavor_name(lv_flavor)) < 0) {
1657 0 : cache = 0;
1658 : }
1659 :
1660 12 : consensus_cache_entry_handle_t *handles[ARRAY_LENGTH(compress_diffs_with)];
1661 12 : memset(handles, 0, sizeof(handles));
1662 :
1663 12 : char description[128];
1664 12 : tor_snprintf(description, sizeof(description),
1665 : "consensus diff from %s to %s",
1666 : lv_from_digest, lv_to_digest);
1667 :
1668 24 : int status = store_multiple(handles,
1669 12 : n_diff_compression_methods(),
1670 : compress_diffs_with,
1671 12 : job->out,
1672 : description);
1673 :
1674 12 : if (status != CDM_DIFF_PRESENT) {
1675 : /* Failure! Nothing to do but complain */
1676 1 : log_warn(LD_DIRSERV,
1677 : "Worker was unable to compute consensus diff "
1678 : "from %s to %s", lv_from_digest, lv_to_digest);
1679 : /* Cache this error so we don't try to compute this one again. */
1680 1 : status = CDM_DIFF_ERROR;
1681 : }
1682 :
1683 12 : unsigned u;
1684 60 : for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
1685 48 : compress_method_t method = compress_diffs_with[u];
1686 48 : if (cache) {
1687 48 : consensus_cache_entry_handle_t *h = handles[u];
1688 48 : int this_status = status;
1689 48 : if (h == NULL) {
1690 4 : this_status = CDM_DIFF_ERROR;
1691 : }
1692 48 : tor_assert_nonfatal(h != NULL || this_status == CDM_DIFF_ERROR);
1693 48 : cdm_diff_ht_set_status(flav, from_sha3, to_sha3, method, this_status, h);
1694 : } else {
1695 48 : consensus_cache_entry_handle_free(handles[u]);
1696 : }
1697 : }
1698 :
1699 12 : consensus_diff_worker_job_free(job);
1700 12 : }
1701 :
1702 : /**
1703 : * Queue the job of computing the diff from <b>diff_from</b> to <b>diff_to</b>
1704 : * in a worker thread.
1705 : */
1706 : static int
1707 15 : consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
1708 : consensus_cache_entry_t *diff_to)
1709 : {
1710 15 : tor_assert(in_main_thread());
1711 :
1712 15 : consensus_cache_entry_incref(diff_from);
1713 15 : consensus_cache_entry_incref(diff_to);
1714 :
1715 15 : consensus_diff_worker_job_t *job = tor_malloc_zero(sizeof(*job));
1716 15 : job->diff_from = diff_from;
1717 15 : job->diff_to = diff_to;
1718 :
1719 : /* Make sure body is mapped. */
1720 15 : const uint8_t *body;
1721 15 : size_t bodylen;
1722 15 : int r1 = consensus_cache_entry_get_body(diff_from, &body, &bodylen);
1723 15 : int r2 = consensus_cache_entry_get_body(diff_to, &body, &bodylen);
1724 15 : if (r1 < 0 || r2 < 0)
1725 0 : goto err;
1726 :
1727 15 : workqueue_entry_t *work;
1728 15 : work = cpuworker_queue_work(WQ_PRI_LOW,
1729 : consensus_diff_worker_threadfn,
1730 : consensus_diff_worker_replyfn,
1731 : job);
1732 15 : if (!work)
1733 0 : goto err;
1734 :
1735 : return 0;
1736 0 : err:
1737 0 : consensus_diff_worker_job_free(job); // includes decrefs.
1738 0 : return -1;
1739 : }
1740 :
1741 : /**
1742 : * Holds requests and replies for consensus_compress_workers.
1743 : */
1744 : typedef struct consensus_compress_worker_job_t {
1745 : char *consensus;
1746 : size_t consensus_len;
1747 : consensus_flavor_t flavor;
1748 : config_line_t *labels_in;
1749 : compressed_result_t out[ARRAY_LENGTH(compress_consensus_with)];
1750 : } consensus_compress_worker_job_t;
1751 :
1752 : #define consensus_compress_worker_job_free(job) \
1753 : FREE_AND_NULL(consensus_compress_worker_job_t, \
1754 : consensus_compress_worker_job_free_, (job))
1755 :
1756 : /**
1757 : * Free all resources held in <b>job</b>
1758 : */
1759 : static void
1760 28 : consensus_compress_worker_job_free_(consensus_compress_worker_job_t *job)
1761 : {
1762 28 : if (!job)
1763 : return;
1764 28 : tor_free(job->consensus);
1765 28 : config_free_lines(job->labels_in);
1766 28 : unsigned u;
1767 112 : for (u = 0; u < n_consensus_compression_methods(); ++u) {
1768 84 : config_free_lines(job->out[u].labels);
1769 84 : tor_free(job->out[u].body);
1770 : }
1771 28 : tor_free(job);
1772 : }
1773 : /**
1774 : * Worker function. This function runs inside a worker thread and receives
1775 : * a consensus_compress_worker_job_t as its input.
1776 : */
1777 : static workqueue_reply_t
1778 28 : consensus_compress_worker_threadfn(void *state_, void *work_)
1779 : {
1780 28 : (void)state_;
1781 28 : consensus_compress_worker_job_t *job = work_;
1782 28 : consensus_flavor_t flavor = job->flavor;
1783 28 : const char *consensus = job->consensus;
1784 28 : size_t bodylen = job->consensus_len;
1785 :
1786 28 : config_line_t *labels = config_lines_dup(job->labels_in);
1787 28 : const char *flavname = networkstatus_get_flavor_name(flavor);
1788 :
1789 28 : cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED,
1790 : (const uint8_t *)consensus, bodylen);
1791 : {
1792 28 : const char *start, *end;
1793 28 : if (router_get_networkstatus_v3_signed_boundaries(consensus, bodylen,
1794 : &start, &end) < 0) {
1795 10 : start = consensus;
1796 10 : end = consensus+bodylen;
1797 : }
1798 28 : cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED,
1799 : (const uint8_t *)start,
1800 28 : end - start);
1801 : }
1802 28 : config_line_prepend(&labels, LABEL_FLAVOR, flavname);
1803 28 : config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
1804 :
1805 28 : compress_multiple(job->out,
1806 28 : n_consensus_compression_methods(),
1807 : compress_consensus_with,
1808 : (const uint8_t*)consensus, bodylen, labels);
1809 28 : config_free_lines(labels);
1810 28 : return WQ_RPL_REPLY;
1811 : }
1812 :
1813 : /**
1814 : * Worker function: This function runs in the main thread, and receives
1815 : * a consensus_diff_compress_job_t that the worker thread has already
1816 : * processed.
1817 : */
1818 : static void
1819 28 : consensus_compress_worker_replyfn(void *work_)
1820 : {
1821 28 : consensus_compress_worker_job_t *job = work_;
1822 :
1823 28 : consensus_cache_entry_handle_t *handles[
1824 : ARRAY_LENGTH(compress_consensus_with)];
1825 28 : memset(handles, 0, sizeof(handles));
1826 :
1827 28 : store_multiple(handles,
1828 28 : n_consensus_compression_methods(),
1829 : compress_consensus_with,
1830 28 : job->out,
1831 : "consensus");
1832 28 : mark_cdm_cache_dirty();
1833 :
1834 28 : unsigned u;
1835 28 : consensus_flavor_t f = job->flavor;
1836 28 : tor_assert((int)f < N_CONSENSUS_FLAVORS);
1837 112 : for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
1838 84 : if (handles[u] == NULL)
1839 0 : continue;
1840 84 : consensus_cache_entry_handle_free(latest_consensus[f][u]);
1841 84 : latest_consensus[f][u] = handles[u];
1842 : }
1843 :
1844 28 : consensus_compress_worker_job_free(job);
1845 28 : }
1846 :
1847 : /**
1848 : * If true, we compress in worker threads.
1849 : */
1850 : static int background_compression = 0;
1851 :
1852 : /**
1853 : * Queue a job to compress <b>consensus</b> and store its compressed
1854 : * text in the cache.
1855 : */
1856 : static int
1857 28 : consensus_queue_compression_work(const char *consensus,
1858 : size_t consensus_len,
1859 : const networkstatus_t *as_parsed)
1860 : {
1861 28 : tor_assert(consensus);
1862 28 : tor_assert(as_parsed);
1863 :
1864 28 : consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job));
1865 28 : job->consensus = tor_memdup_nulterm(consensus, consensus_len);
1866 28 : job->consensus_len = strlen(job->consensus);
1867 28 : job->flavor = as_parsed->flavor;
1868 :
1869 28 : char va_str[ISO_TIME_LEN+1];
1870 28 : char vu_str[ISO_TIME_LEN+1];
1871 28 : char fu_str[ISO_TIME_LEN+1];
1872 28 : format_iso_time_nospace(va_str, as_parsed->valid_after);
1873 28 : format_iso_time_nospace(fu_str, as_parsed->fresh_until);
1874 28 : format_iso_time_nospace(vu_str, as_parsed->valid_until);
1875 28 : config_line_append(&job->labels_in, LABEL_VALID_AFTER, va_str);
1876 28 : config_line_append(&job->labels_in, LABEL_FRESH_UNTIL, fu_str);
1877 28 : config_line_append(&job->labels_in, LABEL_VALID_UNTIL, vu_str);
1878 28 : if (as_parsed->voters) {
1879 1 : smartlist_t *hexvoters = smartlist_new();
1880 1 : SMARTLIST_FOREACH_BEGIN(as_parsed->voters,
1881 : networkstatus_voter_info_t *, vi) {
1882 0 : if (smartlist_len(vi->sigs) == 0)
1883 0 : continue; // didn't sign.
1884 0 : char d[HEX_DIGEST_LEN+1];
1885 0 : base16_encode(d, sizeof(d), vi->identity_digest, DIGEST_LEN);
1886 0 : smartlist_add_strdup(hexvoters, d);
1887 0 : } SMARTLIST_FOREACH_END(vi);
1888 1 : char *signers = smartlist_join_strings(hexvoters, ",", 0, NULL);
1889 1 : config_line_prepend(&job->labels_in, LABEL_SIGNATORIES, signers);
1890 1 : tor_free(signers);
1891 1 : SMARTLIST_FOREACH(hexvoters, char *, cp, tor_free(cp));
1892 1 : smartlist_free(hexvoters);
1893 : }
1894 :
1895 28 : if (background_compression) {
1896 0 : workqueue_entry_t *work;
1897 0 : work = cpuworker_queue_work(WQ_PRI_LOW,
1898 : consensus_compress_worker_threadfn,
1899 : consensus_compress_worker_replyfn,
1900 : job);
1901 0 : if (!work) {
1902 0 : consensus_compress_worker_job_free(job);
1903 0 : return -1;
1904 : }
1905 :
1906 : return 0;
1907 : } else {
1908 28 : consensus_compress_worker_threadfn(NULL, job);
1909 28 : consensus_compress_worker_replyfn(job);
1910 28 : return 0;
1911 : }
1912 : }
1913 :
1914 : /**
1915 : * Tell the consdiffmgr backend to compress consensuses in worker threads.
1916 : */
1917 : void
1918 0 : consdiffmgr_enable_background_compression(void)
1919 : {
1920 : // This isn't the default behavior because it would break unit tests.
1921 0 : background_compression = 1;
1922 0 : }
1923 :
1924 : /** Read the set of voters from the cached object <b>ent</b> into
1925 : * <b>out</b>, as a list of hex-encoded digests. Return 0 on success,
1926 : * -1 if no signatories were recorded. */
1927 : int
1928 1 : consensus_cache_entry_get_voter_id_digests(const consensus_cache_entry_t *ent,
1929 : smartlist_t *out)
1930 : {
1931 1 : tor_assert(ent);
1932 1 : tor_assert(out);
1933 1 : const char *s;
1934 1 : s = consensus_cache_entry_get_value(ent, LABEL_SIGNATORIES);
1935 1 : if (s == NULL)
1936 : return -1;
1937 1 : smartlist_split_string(out, s, ",", SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE, 0);
1938 1 : return 0;
1939 : }
1940 :
1941 : /** Read the fresh-until time of cached object <b>ent</b> into *<b>out</b>
1942 : * and return 0, or return -1 if no such time was recorded. */
1943 : int
1944 5 : consensus_cache_entry_get_fresh_until(const consensus_cache_entry_t *ent,
1945 : time_t *out)
1946 : {
1947 5 : tor_assert(ent);
1948 5 : tor_assert(out);
1949 5 : const char *s;
1950 5 : s = consensus_cache_entry_get_value(ent, LABEL_FRESH_UNTIL);
1951 5 : if (s == NULL || parse_iso_time_nospace(s, out) < 0)
1952 0 : return -1;
1953 : else
1954 : return 0;
1955 : }
1956 :
1957 : /** Read the valid until timestamp from the cached object <b>ent</b> into
1958 : * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
1959 : int
1960 7 : consensus_cache_entry_get_valid_until(const consensus_cache_entry_t *ent,
1961 : time_t *out)
1962 : {
1963 7 : tor_assert(ent);
1964 7 : tor_assert(out);
1965 :
1966 7 : const char *s;
1967 7 : s = consensus_cache_entry_get_value(ent, LABEL_VALID_UNTIL);
1968 7 : if (s == NULL || parse_iso_time_nospace(s, out) < 0)
1969 0 : return -1;
1970 : else
1971 : return 0;
1972 : }
1973 :
1974 : /** Read the valid after timestamp from the cached object <b>ent</b> into
1975 : * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
1976 : int
1977 7 : consensus_cache_entry_get_valid_after(const consensus_cache_entry_t *ent,
1978 : time_t *out)
1979 : {
1980 7 : tor_assert(ent);
1981 7 : tor_assert(out);
1982 :
1983 7 : const char *s;
1984 7 : s = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
1985 :
1986 7 : if (s == NULL || parse_iso_time_nospace(s, out) < 0)
1987 0 : return -1;
1988 : else
1989 : return 0;
1990 : }
|
