Tor  0.4.7.0-alpha-dev
consdiffmgr.c
Go to the documentation of this file.
1 /* Copyright (c) 2017-2021, The Tor Project, Inc. */
2 /* See LICENSE for licensing information */
3 
4 /**
5  * \file consdiffmgr.c
6  *
7  * \brief consensus diff manager functions
8  *
9  * This module is run by directory authorities and caches in order
10  * to remember a number of past consensus documents, and to generate
11  * and serve the diffs from those documents to the latest consensus.
12  */
13 
14 #define CONSDIFFMGR_PRIVATE
15 
16 #include "core/or/or.h"
17 #include "app/config/config.h"
25 #include "lib/evloop/workqueue.h"
26 #include "lib/compress/compress.h"
27 #include "lib/encoding/confline.h"
28 
31 
32 /**
33  * Labels to apply to items in the conscache object.
34  *
35  * @{
36  */
37 /* One of DOCTYPE_CONSENSUS or DOCTYPE_CONSENSUS_DIFF */
38 #define LABEL_DOCTYPE "document-type"
39 /* The valid-after time for a consensus (or for the target consensus of a
40  * diff), encoded as ISO UTC. */
41 #define LABEL_VALID_AFTER "consensus-valid-after"
42 /* The fresh-until time for a consensus (or for the target consensus of a
43  * diff), encoded as ISO UTC. */
44 #define LABEL_FRESH_UNTIL "consensus-fresh-until"
45 /* The valid-until time for a consensus (or for the target consensus of a
46  * diff), encoded as ISO UTC. */
47 #define LABEL_VALID_UNTIL "consensus-valid-until"
48 /* Comma-separated list of hex-encoded identity digests for the voting
49  * authorities. */
50 #define LABEL_SIGNATORIES "consensus-signatories"
51 /* A hex encoded SHA3 digest of the object, as compressed (if any) */
52 #define LABEL_SHA3_DIGEST "sha3-digest"
53 /* A hex encoded SHA3 digest of the object before compression. */
54 #define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed"
55 /* A hex encoded SHA3 digest-as-signed of a consensus */
56 #define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed"
57 /* The flavor of the consensus or consensuses diff */
58 #define LABEL_FLAVOR "consensus-flavor"
59 /* Diff only: the SHA3 digest-as-signed of the source consensus. */
60 #define LABEL_FROM_SHA3_DIGEST "from-sha3-digest"
61 /* Diff only: the SHA3 digest-in-full of the target consensus. */
62 #define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest"
63 /* Diff only: the valid-after date of the source consensus. */
64 #define LABEL_FROM_VALID_AFTER "from-valid-after"
65 /* What kind of compression was used? */
66 #define LABEL_COMPRESSION_TYPE "compression"
67 /** @} */
68 
69 #define DOCTYPE_CONSENSUS "consensus"
70 #define DOCTYPE_CONSENSUS_DIFF "consensus-diff"
71 
72 /**
73  * Underlying directory that stores consensuses and consensus diffs. Don't
74  * use this directly: use cdm_cache_get() instead.
75  */
77 /**
78  * If true, we have learned at least one new consensus since the
79  * consensus cache was last up-to-date.
80  */
81 static int cdm_cache_dirty = 0;
82 /**
83  * If true, we have scanned the cache to update our hashtable of diffs.
84  */
85 static int cdm_cache_loaded = 0;
86 
87 /**
88  * Possible status values for cdm_diff_t.cdm_diff_status
89  **/
90 typedef enum cdm_diff_status_t {
91  CDM_DIFF_PRESENT=1,
92  CDM_DIFF_IN_PROGRESS=2,
93  CDM_DIFF_ERROR=3,
95 
96 /** Which methods do we use for precompressing diffs? */
98  NO_METHOD,
99  GZIP_METHOD,
100 #ifdef HAVE_LZMA
101  LZMA_METHOD,
102 #endif
103 #ifdef HAVE_ZSTD
104  ZSTD_METHOD,
105 #endif
106 };
107 
108 /**
109  * Event for rescanning the cache.
110  */
112 
113 static void consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg);
114 static void mark_cdm_cache_dirty(void);
115 
116 /** How many different methods will we try to use for diff compression? */
117 STATIC unsigned
119 {
121 }
122 
123 /** Which methods do we use for precompressing consensuses? */
125  ZLIB_METHOD,
126 #ifdef HAVE_LZMA
127  LZMA_METHOD,
128 #endif
129 #ifdef HAVE_ZSTD
130  ZSTD_METHOD,
131 #endif
132 };
133 
134 /** How many different methods will we try to use for diff compression? */
135 STATIC unsigned
137 {
139 }
140 
141 /** For which compression method do we retain old consensuses? There's no
142  * need to keep all of them, since we won't be serving them. We'll
143  * go with ZLIB_METHOD because it's pretty fast and everyone has it.
144  */
145 #define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD
146 
147 /** Handles pointing to the latest consensus entries as compressed and
148  * stored. */
149 static consensus_cache_entry_handle_t *
152 
153 /** Hashtable node used to remember the current status of the diff
154  * from a given sha3 digest to the current consensus. */
155 typedef struct cdm_diff_t {
156  HT_ENTRY(cdm_diff_t) node;
157 
158  /** Consensus flavor for this diff (part of ht key) */
159  consensus_flavor_t flavor;
160  /** SHA3-256 digest of the consensus that this diff is _from_. (part of the
161  * ht key) */
162  uint8_t from_sha3[DIGEST256_LEN];
163  /** Method by which the diff is compressed. (part of the ht key */
164  compress_method_t compress_method;
165 
166  /** One of the CDM_DIFF_* values, depending on whether this diff
167  * is available, in progress, or impossible to compute. */
168  cdm_diff_status_t cdm_diff_status;
169  /** SHA3-256 digest of the consensus that this diff is _to. */
170  uint8_t target_sha3[DIGEST256_LEN];
171 
172  /** Handle to the cache entry for this diff, if any. We use a handle here
173  * to avoid thinking too hard about cache entry lifetime issues. */
174  consensus_cache_entry_handle_t *entry;
175 } cdm_diff_t;
176 
177 /** Hashtable mapping flavor and source consensus digest to status. */
178 static HT_HEAD(cdm_diff_ht, cdm_diff_t) cdm_diff_ht = HT_INITIALIZER();
179 
180 #ifdef _WIN32
181  // XXX(ahf): For tor#24857, a contributor suggested that on Windows, the CPU
182  // begins to spike at 100% once the number of files handled by the consensus
183  // diff manager becomes larger than 64. To see if the issue goes away, we
184  // hardcode this value to 64 now while we investigate a better solution.
185 # define CACHE_MAX_NUM 64
186 #else /* !defined(_WIN32) */
187 # define CACHE_MAX_NUM 128
188 #endif /* defined(_WIN32) */
189 
190 /**
191  * Configuration for this module
192  */
193 static consdiff_cfg_t consdiff_cfg = {
194  // XXXX I'd like to make this number bigger, but it interferes with the
195  // XXXX seccomp2 syscall filter, which tops out at BPF_MAXINS (4096)
196  // XXXX rules.
197  /* .cache_max_num = */ CACHE_MAX_NUM
198 };
199 
200 static int consdiffmgr_ensure_space_for_files(int n);
201 static int consensus_queue_compression_work(const char *consensus,
202  size_t consensus_len,
203  const networkstatus_t *as_parsed);
205  consensus_cache_entry_t *diff_to);
206 static void consdiffmgr_set_cache_flags(void);
207 
208 /* =====
209  * Hashtable setup
210  * ===== */
211 
212 /** Helper: hash the key of a cdm_diff_t. */
213 static unsigned
215 {
216  uint8_t tmp[DIGEST256_LEN + 2];
217  memcpy(tmp, diff->from_sha3, DIGEST256_LEN);
218  tmp[DIGEST256_LEN] = (uint8_t) diff->flavor;
219  tmp[DIGEST256_LEN+1] = (uint8_t) diff->compress_method;
220  return (unsigned) siphash24g(tmp, sizeof(tmp));
221 }
222 /** Helper: compare two cdm_diff_t objects for key equality */
223 static int
224 cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2)
225 {
226  return fast_memeq(diff1->from_sha3, diff2->from_sha3, DIGEST256_LEN) &&
227  diff1->flavor == diff2->flavor &&
228  diff1->compress_method == diff2->compress_method;
229 }
230 
231 HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq);
232 HT_GENERATE2(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq,
233  0.6, tor_reallocarray, tor_free_);
234 
235 #define cdm_diff_free(diff) \
236  FREE_AND_NULL(cdm_diff_t, cdm_diff_free_, (diff))
237 
238 /** Release all storage held in <b>diff</b>. */
239 static void
241 {
242  if (!diff)
243  return;
244  consensus_cache_entry_handle_free(diff->entry);
245  tor_free(diff);
246 }
247 
248 /** Create and return a new cdm_diff_t with the given values. Does not
249  * add it to the hashtable. */
250 static cdm_diff_t *
252  const uint8_t *from_sha3,
253  const uint8_t *target_sha3,
254  compress_method_t method)
255 {
256  cdm_diff_t *ent;
257  ent = tor_malloc_zero(sizeof(cdm_diff_t));
258  ent->flavor = flav;
259  memcpy(ent->from_sha3, from_sha3, DIGEST256_LEN);
260  memcpy(ent->target_sha3, target_sha3, DIGEST256_LEN);
261  ent->compress_method = method;
262  return ent;
263 }
264 
265 /**
266  * Examine the diff hashtable to see whether we know anything about computing
267  * a diff of type <b>flav</b> between consensuses with the two provided
268  * SHA3-256 digests. If a computation is in progress, or if the computation
269  * has already been tried and failed, return 1. Otherwise, note the
270  * computation as "in progress" so that we don't reattempt it later, and
271  * return 0.
272  */
273 static int
275  const uint8_t *from_sha3,
276  const uint8_t *target_sha3)
277 {
278  struct cdm_diff_t search, *ent;
279  unsigned u;
280  int result = 0;
281  for (u = 0; u < n_diff_compression_methods(); ++u) {
283  memset(&search, 0, sizeof(cdm_diff_t));
284  search.flavor = flav;
285  search.compress_method = method;
286  memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
287  ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
288  if (ent) {
289  tor_assert_nonfatal(ent->cdm_diff_status != CDM_DIFF_PRESENT);
290  result = 1;
291  continue;
292  }
293  ent = cdm_diff_new(flav, from_sha3, target_sha3, method);
294  ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
295  HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
296  }
297  return result;
298 }
299 
300 /**
301  * Update the status of the diff of type <b>flav</b> between consensuses with
302  * the two provided SHA3-256 digests, so that its status becomes
303  * <b>status</b>, and its value becomes the <b>handle</b>. If <b>handle</b>
304  * is NULL, then the old handle (if any) is freed, and replaced with NULL.
305  */
306 static void
308  const uint8_t *from_sha3,
309  const uint8_t *to_sha3,
310  compress_method_t method,
311  int status,
312  consensus_cache_entry_handle_t *handle)
313 {
314  if (handle == NULL) {
315  tor_assert_nonfatal(status != CDM_DIFF_PRESENT);
316  }
317 
318  struct cdm_diff_t search, *ent;
319  memset(&search, 0, sizeof(cdm_diff_t));
320  search.flavor = flav;
321  search.compress_method = method,
322  memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
323  ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
324  if (!ent) {
325  ent = cdm_diff_new(flav, from_sha3, to_sha3, method);
326  ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
327  HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
328  } else if (fast_memneq(ent->target_sha3, to_sha3, DIGEST256_LEN)) {
329  // This can happen under certain really pathological conditions
330  // if we decide we don't care about a diff before it is actually
331  // done computing.
332  return;
333  }
334 
335  tor_assert_nonfatal(ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS);
336 
337  ent->cdm_diff_status = status;
338  consensus_cache_entry_handle_free(ent->entry);
339  ent->entry = handle;
340 }
341 
342 /**
343  * Helper: Remove from the hash table every present (actually computed) diff
344  * of type <b>flav</b> whose target digest does not match
345  * <b>unless_target_sha3_matches</b>.
346  *
347  * This function is used for the hash table to throw away references to diffs
348  * that do not lead to the most given consensus of a given flavor.
349  */
350 static void
352  const uint8_t *unless_target_sha3_matches)
353 {
354  cdm_diff_t **diff, **next;
355  for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
356  cdm_diff_t *this = *diff;
357 
358  if ((*diff)->cdm_diff_status == CDM_DIFF_PRESENT &&
359  flav == (*diff)->flavor) {
360 
361  if (BUG((*diff)->entry == NULL) ||
362  consensus_cache_entry_handle_get((*diff)->entry) == NULL) {
363  /* the underlying entry has gone away; drop this. */
364  next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
365  cdm_diff_free(this);
366  continue;
367  }
368 
369  if (unless_target_sha3_matches &&
370  fast_memneq(unless_target_sha3_matches, (*diff)->target_sha3,
371  DIGEST256_LEN)) {
372  /* target hash doesn't match; drop this. */
373  next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
374  cdm_diff_free(this);
375  continue;
376  }
377  }
378  next = HT_NEXT(cdm_diff_ht, &cdm_diff_ht, diff);
379  }
380 }
381 
382 /**
383  * Helper: initialize <b>cons_diff_cache</b>.
384  */
385 static void
387 {
388  unsigned n_entries = consdiff_cfg.cache_max_num * 2;
389 
390  tor_assert(cons_diff_cache == NULL);
391  cons_diff_cache = consensus_cache_open("diff-cache", n_entries);
392  if (cons_diff_cache == NULL) {
393  // LCOV_EXCL_START
394  log_err(LD_FS, "Error: Couldn't open storage for consensus diffs.");
395  tor_assert_unreached();
396  // LCOV_EXCL_STOP
397  } else {
399  }
403  cdm_cache_loaded = 0;
404 }
405 
406 /**
407  * Helper: return the consensus_cache_t * that backs this manager,
408  * initializing it if needed.
409  */
412 {
413  if (PREDICT_UNLIKELY(cons_diff_cache == NULL)) {
414  cdm_cache_init();
415  }
416  return cons_diff_cache;
417 }
418 
419 /**
420  * Helper: given a list of labels, prepend the hex-encoded SHA3 digest
421  * of the <b>bodylen</b>-byte object at <b>body</b> to those labels,
422  * with <b>label</b> as its label.
423  */
424 static void
426  const char *label,
427  const uint8_t *body,
428  size_t bodylen)
429 {
430  uint8_t sha3_digest[DIGEST256_LEN];
431  char hexdigest[HEX_DIGEST256_LEN+1];
432  crypto_digest256((char *)sha3_digest,
433  (const char *)body, bodylen, DIGEST_SHA3_256);
434  base16_encode(hexdigest, sizeof(hexdigest),
435  (const char *)sha3_digest, sizeof(sha3_digest));
436 
437  config_line_prepend(labels, label, hexdigest);
438 }
439 
440 /** Helper: if there is a sha3-256 hex-encoded digest in <b>ent</b> with the
441  * given label, set <b>digest_out</b> to that value (decoded), and return 0.
442  *
443  * Return -1 if there is no such label, and -2 if it is badly formatted. */
444 STATIC int
445 cdm_entry_get_sha3_value(uint8_t *digest_out,
447  const char *label)
448 {
449  if (ent == NULL)
450  return -1;
451 
452  const char *hex = consensus_cache_entry_get_value(ent, label);
453  if (hex == NULL)
454  return -1;
455 
456  int n = base16_decode((char*)digest_out, DIGEST256_LEN, hex, strlen(hex));
457  if (n != DIGEST256_LEN)
458  return -2;
459  else
460  return 0;
461 }
462 
463 /**
464  * Helper: look for a consensus with the given <b>flavor</b> and
465  * <b>valid_after</b> time in the cache. Return that consensus if it's
466  * present, or NULL if it's missing.
467  */
470 {
471  char formatted_time[ISO_TIME_LEN+1];
472  format_iso_time_nospace(formatted_time, valid_after);
473  const char *flavname = networkstatus_get_flavor_name(flavor);
474 
475  /* We'll filter by valid-after time first, since that should
476  * match the fewest documents. */
477  /* We could add an extra hashtable here, but since we only do this scan
478  * when adding a new consensus, it probably doesn't matter much. */
479  smartlist_t *matches = smartlist_new();
481  LABEL_VALID_AFTER, formatted_time);
482  consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
483  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
484 
485  consensus_cache_entry_t *result = NULL;
486  if (smartlist_len(matches)) {
487  result = smartlist_get(matches, 0);
488  }
489  smartlist_free(matches);
490 
491  return result;
492 }
493 
494 /** Return the maximum age (in seconds) of consensuses that we should consider
495  * storing. The available space in the directory may impose additional limits
496  * on how much we store. */
497 static int32_t
499 {
500  const int32_t DEFAULT_MAX_AGE_TO_CACHE = 8192;
501  const int32_t MIN_MAX_AGE_TO_CACHE = 0;
502  const int32_t MAX_MAX_AGE_TO_CACHE = 8192;
503  const char MAX_AGE_TO_CACHE_NAME[] = "max-consensus-age-to-cache-for-diff";
504 
505  const or_options_t *options = get_options();
506 
507  if (options->MaxConsensusAgeForDiffs) {
508  const int v = options->MaxConsensusAgeForDiffs;
509  if (v >= MAX_MAX_AGE_TO_CACHE * 3600)
510  return MAX_MAX_AGE_TO_CACHE;
511  else
512  return v;
513  }
514 
515  /* The parameter is in hours, so we multiply */
516  return 3600 * networkstatus_get_param(NULL,
517  MAX_AGE_TO_CACHE_NAME,
518  DEFAULT_MAX_AGE_TO_CACHE,
519  MIN_MAX_AGE_TO_CACHE,
520  MAX_MAX_AGE_TO_CACHE);
521 }
522 
523 #ifdef TOR_UNIT_TESTS
524 /** As consdiffmgr_add_consensus, but requires a nul-terminated input. For
525  * testing. */
526 int
527 consdiffmgr_add_consensus_nulterm(const char *consensus,
528  const networkstatus_t *as_parsed)
529 {
530  size_t len = strlen(consensus);
531  /* make a non-nul-terminated copy so that we can have a better chance
532  * of catching errors. */
533  char *ctmp = tor_memdup(consensus, len);
534  int r = consdiffmgr_add_consensus(ctmp, len, as_parsed);
535  tor_free(ctmp);
536  return r;
537 }
538 #endif /* defined(TOR_UNIT_TESTS) */
539 
540 /**
541  * Given a buffer containing a networkstatus consensus, and the results of
542  * having parsed that consensus, add that consensus to the cache if it is not
543  * already present and not too old. Create new consensus diffs from or to
544  * that consensus as appropriate.
545  *
546  * Return 0 on success and -1 on failure.
547  */
548 int
549 consdiffmgr_add_consensus(const char *consensus,
550  size_t consensus_len,
551  const networkstatus_t *as_parsed)
552 {
553  if (BUG(consensus == NULL) || BUG(as_parsed == NULL))
554  return -1; // LCOV_EXCL_LINE
555  if (BUG(as_parsed->type != NS_TYPE_CONSENSUS))
556  return -1; // LCOV_EXCL_LINE
557 
558  const consensus_flavor_t flavor = as_parsed->flavor;
559  const time_t valid_after = as_parsed->valid_after;
560 
561  if (valid_after < approx_time() - get_max_age_to_cache()) {
562  log_info(LD_DIRSERV, "We don't care about this consensus document; it's "
563  "too old.");
564  return -1;
565  }
566 
567  /* Do we already have this one? */
568  consensus_cache_entry_t *entry =
569  cdm_cache_lookup_consensus(flavor, valid_after);
570  if (entry) {
571  log_info(LD_DIRSERV, "We already have a copy of that consensus");
572  return -1;
573  }
574 
575  /* We don't have it. Add it to the cache. */
576  return consensus_queue_compression_work(consensus, consensus_len, as_parsed);
577 }
578 
579 /**
580  * Helper: used to sort two smartlists of consensus_cache_entry_t by their
581  * LABEL_VALID_AFTER labels.
582  */
583 static int
584 compare_by_valid_after_(const void **a, const void **b)
585 {
586  const consensus_cache_entry_t *e1 = *a;
587  const consensus_cache_entry_t *e2 = *b;
588  /* We're in luck here: sorting UTC iso-encoded values lexically will work
589  * fine (until 9999). */
590  return strcmp_opt(consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER),
591  consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER));
592 }
593 
594 /**
595  * Helper: Sort <b>lst</b> by LABEL_VALID_AFTER and return the most recent
596  * entry.
597  */
600 {
602  if (smartlist_len(lst)) {
603  return smartlist_get(lst, smartlist_len(lst) - 1);
604  } else {
605  return NULL;
606  }
607 }
608 
609 /** Return i such that compress_consensus_with[i] == method. Return
610  * -1 if no such i exists. */
611 static int
613 {
614  unsigned i;
615  for (i = 0; i < n_consensus_compression_methods(); ++i) {
616  if (compress_consensus_with[i] == method) {
617  return i;
618  }
619  }
620  return -1;
621 }
622 
623 /**
624  * If we know a consensus with the flavor <b>flavor</b> compressed with
625  * <b>method</b>, set *<b>entry_out</b> to that value. Return values are as
626  * for consdiffmgr_find_diff_from().
627  */
630  consensus_flavor_t flavor,
631  compress_method_t method)
632 {
633  tor_assert(entry_out);
634  tor_assert((int)flavor < N_CONSENSUS_FLAVORS);
635 
636  int pos = consensus_compression_method_pos(method);
637  if (pos < 0) {
638  // We don't compress consensuses with this method.
639  return CONSDIFF_NOT_FOUND;
640  }
641  consensus_cache_entry_handle_t *handle = latest_consensus[flavor][pos];
642  if (!handle)
643  return CONSDIFF_NOT_FOUND;
644  *entry_out = consensus_cache_entry_handle_get(handle);
645  if (*entry_out)
646  return CONSDIFF_AVAILABLE;
647  else
648  return CONSDIFF_NOT_FOUND;
649 }
650 
651 /**
652  * Look up consensus_cache_entry_t for the consensus of type <b>flavor</b>,
653  * from the source consensus with the specified digest (which must be SHA3).
654  *
655  * If the diff is present, store it into *<b>entry_out</b> and return
656  * CONSDIFF_AVAILABLE. Otherwise return CONSDIFF_NOT_FOUND or
657  * CONSDIFF_IN_PROGRESS.
658  */
661  consensus_flavor_t flavor,
662  int digest_type,
663  const uint8_t *digest,
664  size_t digestlen,
665  compress_method_t method)
666 {
667  if (BUG(digest_type != DIGEST_SHA3_256) ||
668  BUG(digestlen != DIGEST256_LEN)) {
669  return CONSDIFF_NOT_FOUND; // LCOV_EXCL_LINE
670  }
671 
672  // Try to look up the entry in the hashtable.
673  cdm_diff_t search, *ent;
674  memset(&search, 0, sizeof(search));
675  search.flavor = flavor;
676  search.compress_method = method;
677  memcpy(search.from_sha3, digest, DIGEST256_LEN);
678  ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
679 
680  if (ent == NULL ||
681  ent->cdm_diff_status == CDM_DIFF_ERROR) {
682  return CONSDIFF_NOT_FOUND;
683  } else if (ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS) {
684  return CONSDIFF_IN_PROGRESS;
685  } else if (BUG(ent->cdm_diff_status != CDM_DIFF_PRESENT)) {
686  return CONSDIFF_IN_PROGRESS;
687  }
688 
689  if (BUG(ent->entry == NULL)) {
690  return CONSDIFF_NOT_FOUND;
691  }
692  *entry_out = consensus_cache_entry_handle_get(ent->entry);
693  return (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
694 
695 #if 0
696  // XXXX Remove this. I'm keeping it around for now in case we need to
697  // XXXX debug issues in the hashtable.
698  char hex[HEX_DIGEST256_LEN+1];
699  base16_encode(hex, sizeof(hex), (const char *)digest, digestlen);
700  const char *flavname = networkstatus_get_flavor_name(flavor);
701 
702  smartlist_t *matches = smartlist_new();
704  LABEL_FROM_SHA3_DIGEST, hex);
705  consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
706  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
707 
708  *entry_out = sort_and_find_most_recent(matches);
709  consdiff_status_t result =
710  (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
711  smartlist_free(matches);
712 
713  return result;
714 #endif /* 0 */
715 }
716 
717 /**
718  * Perform periodic cleanup tasks on the consensus diff cache. Return
719  * the number of objects marked for deletion.
720  */
721 int
723 {
724  smartlist_t *objects = smartlist_new();
725  smartlist_t *consensuses = smartlist_new();
726  smartlist_t *diffs = smartlist_new();
727  int n_to_delete = 0;
728 
729  log_debug(LD_DIRSERV, "Looking for consdiffmgr entries to remove");
730 
731  // 1. Delete any consensus or diff or anything whose valid_after is too old.
732  const time_t valid_after_cutoff = approx_time() - get_max_age_to_cache();
733 
735  NULL, NULL);
737  const char *lv_valid_after =
738  consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
739  if (! lv_valid_after) {
740  log_debug(LD_DIRSERV, "Ignoring entry because it had no %s label",
741  LABEL_VALID_AFTER);
742  continue;
743  }
744  time_t valid_after = 0;
745  if (parse_iso_time_nospace(lv_valid_after, &valid_after) < 0) {
746  log_debug(LD_DIRSERV, "Ignoring entry because its %s value (%s) was "
747  "unparseable", LABEL_VALID_AFTER, escaped(lv_valid_after));
748  continue;
749  }
750  if (valid_after < valid_after_cutoff) {
751  log_debug(LD_DIRSERV, "Deleting entry because its %s value (%s) was "
752  "too old", LABEL_VALID_AFTER, lv_valid_after);
754  ++n_to_delete;
755  }
756  } SMARTLIST_FOREACH_END(ent);
757 
758  // 2. Delete all diffs that lead to a consensus whose valid-after is not the
759  // latest.
760  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
761  const char *flavname = networkstatus_get_flavor_name(flav);
762  /* Determine the most recent consensus of this flavor */
764  LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
765  consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
766  consensus_cache_entry_t *most_recent =
767  sort_and_find_most_recent(consensuses);
768  if (most_recent == NULL)
769  continue;
770  const char *most_recent_sha3 =
772  LABEL_SHA3_DIGEST_UNCOMPRESSED);
773  if (BUG(most_recent_sha3 == NULL))
774  continue; // LCOV_EXCL_LINE
775 
776  /* consider all such-flavored diffs, and look to see if they match. */
778  LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
779  consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
781  const char *this_diff_target_sha3 =
782  consensus_cache_entry_get_value(diff, LABEL_TARGET_SHA3_DIGEST);
783  if (!this_diff_target_sha3)
784  continue;
785  if (strcmp(this_diff_target_sha3, most_recent_sha3)) {
787  ++n_to_delete;
788  }
789  } SMARTLIST_FOREACH_END(diff);
790  smartlist_clear(consensuses);
791  smartlist_clear(diffs);
792  }
793 
794  // 3. Delete all consensuses except the most recent that are compressed with
795  // an un-preferred method.
796  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
797  const char *flavname = networkstatus_get_flavor_name(flav);
798  /* Determine the most recent consensus of this flavor */
800  LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
801  consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
802  consensus_cache_entry_t *most_recent =
803  sort_and_find_most_recent(consensuses);
804  if (most_recent == NULL)
805  continue;
806  const char *most_recent_sha3_uncompressed =
808  LABEL_SHA3_DIGEST_UNCOMPRESSED);
809  const char *retain_methodname = compression_method_get_name(
811 
812  if (BUG(most_recent_sha3_uncompressed == NULL))
813  continue;
814  SMARTLIST_FOREACH_BEGIN(consensuses, consensus_cache_entry_t *, ent) {
815  const char *lv_sha3_uncompressed =
816  consensus_cache_entry_get_value(ent, LABEL_SHA3_DIGEST_UNCOMPRESSED);
817  if (BUG(! lv_sha3_uncompressed))
818  continue;
819  if (!strcmp(lv_sha3_uncompressed, most_recent_sha3_uncompressed))
820  continue; // This _is_ the most recent.
821  const char *lv_methodname =
822  consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
823  if (! lv_methodname || strcmp(lv_methodname, retain_methodname)) {
825  ++n_to_delete;
826  }
827  } SMARTLIST_FOREACH_END(ent);
828  }
829 
830  smartlist_free(objects);
831  smartlist_free(consensuses);
832  smartlist_free(diffs);
833 
834  // Actually remove files, if they're not used.
836  return n_to_delete;
837 }
838 
839 /**
840  * Initialize the consensus diff manager and its cache, and configure
841  * its parameters based on the latest torrc and networkstatus parameters.
842  */
843 void
845 {
846  if (cfg)
847  memcpy(&consdiff_cfg, cfg, sizeof(consdiff_cfg));
848 
849  (void) cdm_cache_get();
850 }
851 
852 /**
853  * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
854  * operations that the consensus diff manager will need.
855  */
856 int
857 consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem_t **cfg)
858 {
860 }
861 
862 /**
863  * Scan the consensus diff manager's cache for any grossly malformed entries,
864  * and mark them as deletable. Return 0 if no problems were found; 1
865  * if problems were found and fixed.
866  */
867 int
869 {
870  /* Right now, we only check for entries that have bad sha3 values */
871  int problems = 0;
872 
873  smartlist_t *objects = smartlist_new();
875  NULL, NULL);
877  uint8_t sha3_expected[DIGEST256_LEN];
878  uint8_t sha3_received[DIGEST256_LEN];
879  int r = cdm_entry_get_sha3_value(sha3_expected, obj, LABEL_SHA3_DIGEST);
880  if (r == -1) {
881  /* digest isn't there; that's allowed */
882  continue;
883  } else if (r == -2) {
884  /* digest is malformed; that's not allowed */
885  problems = 1;
887  continue;
888  }
889  const uint8_t *body;
890  size_t bodylen;
892  r = consensus_cache_entry_get_body(obj, &body, &bodylen);
893  if (r == 0) {
894  crypto_digest256((char *)sha3_received, (const char *)body, bodylen,
895  DIGEST_SHA3_256);
896  }
898  if (r < 0)
899  continue;
900 
901  // Deconfuse coverity about the possibility of sha3_received being
902  // uninitialized
903  tor_assert(r <= 0);
904 
905  if (fast_memneq(sha3_received, sha3_expected, DIGEST256_LEN)) {
906  problems = 1;
908  continue;
909  }
910 
911  } SMARTLIST_FOREACH_END(obj);
912  smartlist_free(objects);
913  return problems;
914 }
915 
916 /**
917  * Helper: build new diffs of <b>flavor</b> as needed
918  */
919 static void
921 {
922  smartlist_t *matches = NULL;
923  smartlist_t *diffs = NULL;
924  smartlist_t *compute_diffs_from = NULL;
925  strmap_t *have_diff_from = NULL;
926 
927  // look for the most recent consensus, and for all previous in-range
928  // consensuses. Do they all have diffs to it?
929  const char *flavname = networkstatus_get_flavor_name(flavor);
930 
931  // 1. find the most recent consensus, and the ones that we might want
932  // to diff to it.
933  const char *methodname = compression_method_get_name(
935 
936  matches = smartlist_new();
938  LABEL_FLAVOR, flavname);
939  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
940  consensus_cache_filter_list(matches, LABEL_COMPRESSION_TYPE, methodname);
941  consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
942  if (!most_recent) {
943  log_info(LD_DIRSERV, "No 'most recent' %s consensus found; "
944  "not making diffs", flavname);
945  goto done;
946  }
947  tor_assert(smartlist_len(matches));
948  smartlist_del(matches, smartlist_len(matches) - 1);
949 
950  const char *most_recent_valid_after =
951  consensus_cache_entry_get_value(most_recent, LABEL_VALID_AFTER);
952  if (BUG(most_recent_valid_after == NULL))
953  goto done; //LCOV_EXCL_LINE
954  uint8_t most_recent_sha3[DIGEST256_LEN];
955  if (BUG(cdm_entry_get_sha3_value(most_recent_sha3, most_recent,
956  LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
957  goto done; //LCOV_EXCL_LINE
958 
959  // 2. Find all the relevant diffs _to_ this consensus. These are ones
960  // that we don't need to compute.
961  diffs = smartlist_new();
963  LABEL_VALID_AFTER, most_recent_valid_after);
964  consensus_cache_filter_list(diffs, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
965  consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
966  have_diff_from = strmap_new();
968  const char *va = consensus_cache_entry_get_value(diff,
969  LABEL_FROM_VALID_AFTER);
970  if (BUG(va == NULL))
971  continue; // LCOV_EXCL_LINE
972  strmap_set(have_diff_from, va, diff);
973  } SMARTLIST_FOREACH_END(diff);
974 
975  // 3. See which consensuses in 'matches' don't have diffs yet.
976  smartlist_reverse(matches); // from newest to oldest.
977  compute_diffs_from = smartlist_new();
979  const char *va = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
980  if (BUG(va == NULL))
981  continue; // LCOV_EXCL_LINE
982  if (strmap_get(have_diff_from, va) != NULL)
983  continue; /* we already have this one. */
984  smartlist_add(compute_diffs_from, ent);
985  /* Since we are not going to serve this as the most recent consensus
986  * any more, we should stop keeping it mmap'd when it's not in use.
987  */
989  } SMARTLIST_FOREACH_END(ent);
990 
991  log_info(LD_DIRSERV,
992  "The most recent %s consensus is valid-after %s. We have diffs to "
993  "this consensus for %d/%d older %s consensuses. Generating diffs "
994  "for the other %d.",
995  flavname,
996  most_recent_valid_after,
997  smartlist_len(matches) - smartlist_len(compute_diffs_from),
998  smartlist_len(matches),
999  flavname,
1000  smartlist_len(compute_diffs_from));
1001 
1002  // 4. Update the hashtable; remove entries in this flavor to other
1003  // target consensuses.
1004  cdm_diff_ht_purge(flavor, most_recent_sha3);
1005 
1006  // 5. Actually launch the requests.
1007  SMARTLIST_FOREACH_BEGIN(compute_diffs_from, consensus_cache_entry_t *, c) {
1008  if (BUG(c == most_recent))
1009  continue; // LCOV_EXCL_LINE
1010 
1011  uint8_t this_sha3[DIGEST256_LEN];
1012  if (cdm_entry_get_sha3_value(this_sha3, c,
1013  LABEL_SHA3_DIGEST_AS_SIGNED)<0) {
1014  // Not actually a bug, since we might be running with a directory
1015  // with stale files from before the #22143 fixes.
1016  continue;
1017  }
1019  this_sha3, most_recent_sha3)) {
1020  // This is already pending, or we encountered an error.
1021  continue;
1022  }
1023  consensus_diff_queue_diff_work(c, most_recent);
1024  } SMARTLIST_FOREACH_END(c);
1025 
1026  done:
1027  smartlist_free(matches);
1028  smartlist_free(diffs);
1029  smartlist_free(compute_diffs_from);
1030  strmap_free(have_diff_from, NULL);
1031 }
1032 
1033 /**
1034  * Scan the cache for the latest consensuses and add their handles to
1035  * latest_consensus
1036  */
1037 static void
1039 {
1040  smartlist_t *matches = smartlist_new();
1041  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
1042  const char *flavname = networkstatus_get_flavor_name(flav);
1043  smartlist_clear(matches);
1045  LABEL_FLAVOR, flavname);
1046  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
1047  consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
1048  if (! most_recent)
1049  continue; // no consensuses.
1050  const char *most_recent_sha3 =
1051  consensus_cache_entry_get_value(most_recent,
1052  LABEL_SHA3_DIGEST_UNCOMPRESSED);
1053  if (BUG(most_recent_sha3 == NULL))
1054  continue; // LCOV_EXCL_LINE
1055  consensus_cache_filter_list(matches, LABEL_SHA3_DIGEST_UNCOMPRESSED,
1056  most_recent_sha3);
1057 
1058  // Everything that remains matches the most recent consensus of this
1059  // flavor.
1061  const char *lv_compression =
1062  consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
1063  compress_method_t method =
1064  compression_method_get_by_name(lv_compression);
1065  int pos = consensus_compression_method_pos(method);
1066  if (pos < 0)
1067  continue;
1068  consensus_cache_entry_handle_free(latest_consensus[flav][pos]);
1069  latest_consensus[flav][pos] = consensus_cache_entry_handle_new(ent);
1070  } SMARTLIST_FOREACH_END(ent);
1071  }
1072  smartlist_free(matches);
1073 }
1074 
1075 /**
1076  * Scan the cache for diffs, and add them to the hashtable.
1077  */
1078 static void
1080 {
1081  smartlist_t *diffs = smartlist_new();
1083  LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
1085  const char *lv_flavor =
1086  consensus_cache_entry_get_value(diff, LABEL_FLAVOR);
1087  if (!lv_flavor)
1088  continue;
1089  int flavor = networkstatus_parse_flavor_name(lv_flavor);
1090  if (flavor < 0)
1091  continue;
1092  const char *lv_compression =
1093  consensus_cache_entry_get_value(diff, LABEL_COMPRESSION_TYPE);
1094  compress_method_t method = NO_METHOD;
1095  if (lv_compression) {
1096  method = compression_method_get_by_name(lv_compression);
1097  if (method == UNKNOWN_METHOD) {
1098  continue;
1099  }
1100  }
1101 
1102  uint8_t from_sha3[DIGEST256_LEN];
1103  uint8_t to_sha3[DIGEST256_LEN];
1104  if (cdm_entry_get_sha3_value(from_sha3, diff, LABEL_FROM_SHA3_DIGEST)<0)
1105  continue;
1106  if (cdm_entry_get_sha3_value(to_sha3, diff, LABEL_TARGET_SHA3_DIGEST)<0)
1107  continue;
1108 
1109  cdm_diff_ht_set_status(flavor, from_sha3, to_sha3,
1110  method,
1111  CDM_DIFF_PRESENT,
1112  consensus_cache_entry_handle_new(diff));
1113  } SMARTLIST_FOREACH_END(diff);
1114  smartlist_free(diffs);
1115 }
1116 
1117 /**
1118  * Build new diffs as needed.
1119  */
1120 void
1122 {
1123  if (cdm_cache_dirty == 0)
1124  return;
1125 
1126  // Clean up here to make room for new diffs, and to ensure that older
1127  // consensuses do not have any entries.
1129 
1130  if (cdm_cache_loaded == 0) {
1133  cdm_cache_loaded = 1;
1134  }
1135 
1136  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
1138  }
1139 
1140  cdm_cache_dirty = 0;
1141 }
1142 
1143 /** Callback wrapper for consdiffmgr_rescan */
1144 static void
1146 {
1147  (void)ev;
1148  (void)arg;
1150 }
1151 
1152 /** Mark the cache as dirty, and schedule a rescan event. */
1153 static void
1155 {
1156  cdm_cache_dirty = 1;
1159 }
1160 
1161 /**
1162  * Helper: compare two files by their from-valid-after and valid-after labels,
1163  * trying to sort in ascending order by from-valid-after (when present) and
1164  * valid-after (when not). Place everything that has neither label first in
1165  * the list.
1166  */
1167 static int
1168 compare_by_staleness_(const void **a, const void **b)
1169 {
1170  const consensus_cache_entry_t *e1 = *a;
1171  const consensus_cache_entry_t *e2 = *b;
1172  const char *va1, *fva1, *va2, *fva2;
1173  va1 = consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER);
1174  va2 = consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER);
1175  fva1 = consensus_cache_entry_get_value(e1, LABEL_FROM_VALID_AFTER);
1176  fva2 = consensus_cache_entry_get_value(e2, LABEL_FROM_VALID_AFTER);
1177 
1178  if (fva1)
1179  va1 = fva1;
1180  if (fva2)
1181  va2 = fva2;
1182 
1183  /* See note about iso-encoded values in compare_by_valid_after_. Also note
1184  * that missing dates will get placed first. */
1185  return strcmp_opt(va1, va2);
1186 }
1187 
1188 /** If there are not enough unused filenames to store <b>n</b> files, then
1189  * delete old consensuses until there are. (We have to keep track of the
1190  * number of filenames because of the way that the seccomp2 cache works.)
1191  *
1192  * Return 0 on success, -1 on failure.
1193  **/
1194 static int
1196 {
1197  consensus_cache_t *cache = cdm_cache_get();
1199  // there are already enough unused filenames.
1200  return 0;
1201  }
1202  // Try a cheap deletion of stuff that's waiting to get deleted.
1205  // okay, _that_ made enough filenames available.
1206  return 0;
1207  }
1208  // Let's get more assertive: clean out unused stuff, and force-remove
1209  // the files that we can.
1212  const int n_to_remove = n - consensus_cache_get_n_filenames_available(cache);
1213  if (n_to_remove <= 0) {
1214  // okay, finally!
1215  return 0;
1216  }
1217 
1218  // At this point, we're going to have to throw out objects that will be
1219  // missed. Too bad!
1220  smartlist_t *objects = smartlist_new();
1221  consensus_cache_find_all(objects, cache, NULL, NULL);
1223  int n_marked = 0;
1226  if (++n_marked >= n_to_remove)
1227  break;
1228  } SMARTLIST_FOREACH_END(ent);
1229  smartlist_free(objects);
1230 
1232 
1233  if (consensus_cache_may_overallocate(cache)) {
1234  /* If we're allowed to throw extra files into the cache, let's do so
1235  * rather getting upset.
1236  */
1237  return 0;
1238  }
1239 
1240  if (BUG(n_marked < n_to_remove))
1241  return -1;
1242  else
1243  return 0;
1244 }
1245 
1246 /**
1247  * Set consensus cache flags on the objects in this consdiffmgr.
1248  */
1249 static void
1251 {
1252  /* Right now, we just mark the consensus objects for aggressive release,
1253  * so that they get mmapped for as little time as possible. */
1254  smartlist_t *objects = smartlist_new();
1256  DOCTYPE_CONSENSUS);
1259  } SMARTLIST_FOREACH_END(ent);
1260  smartlist_free(objects);
1261 }
1262 
1263 /**
1264  * Called before shutdown: drop all storage held by the consdiffmgr.c module.
1265  */
1266 void
1268 {
1269  cdm_diff_t **diff, **next;
1270  for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
1271  cdm_diff_t *this = *diff;
1272  next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
1273  cdm_diff_free(this);
1274  }
1275  int i;
1276  unsigned j;
1277  for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
1278  for (j = 0; j < n_consensus_compression_methods(); ++j) {
1279  consensus_cache_entry_handle_free(latest_consensus[i][j]);
1280  }
1281  }
1282  memset(latest_consensus, 0, sizeof(latest_consensus));
1283  consensus_cache_free(cons_diff_cache);
1284  cons_diff_cache = NULL;
1285  mainloop_event_free(consdiffmgr_rescan_ev);
1286 }
1287 
1288 /* =====
1289  Thread workers
1290  =====*/
1291 
1292 typedef struct compressed_result_t {
1293  config_line_t *labels;
1294  /**
1295  * Output: Body of the diff, as compressed.
1296  */
1297  uint8_t *body;
1298  /**
1299  * Output: length of body_out
1300  */
1301  size_t bodylen;
1303 
1304 /**
1305  * Compress the bytestring <b>input</b> of length <b>len</b> using the
1306  * <b>n_methods</b> compression methods listed in the array <b>methods</b>.
1307  *
1308  * For each successful compression, set the fields in the <b>results_out</b>
1309  * array in the position corresponding to the compression method. Use
1310  * <b>labels_in</b> as a basis for the labels of the result.
1311  *
1312  * Return 0 if all compression succeeded; -1 if any failed.
1313  */
1314 static int
1315 compress_multiple(compressed_result_t *results_out, int n_methods,
1316  const compress_method_t *methods,
1317  const uint8_t *input, size_t len,
1318  const config_line_t *labels_in)
1319 {
1320  int rv = 0;
1321  int i;
1322  for (i = 0; i < n_methods; ++i) {
1323  compress_method_t method = methods[i];
1324  const char *methodname = compression_method_get_name(method);
1325  char *result;
1326  size_t sz;
1327  if (0 == tor_compress(&result, &sz, (const char*)input, len, method)) {
1328  results_out[i].body = (uint8_t*)result;
1329  results_out[i].bodylen = sz;
1330  results_out[i].labels = config_lines_dup(labels_in);
1331  cdm_labels_prepend_sha3(&results_out[i].labels, LABEL_SHA3_DIGEST,
1332  results_out[i].body,
1333  results_out[i].bodylen);
1334  config_line_prepend(&results_out[i].labels,
1335  LABEL_COMPRESSION_TYPE,
1336  methodname);
1337  } else {
1338  rv = -1;
1339  }
1340  }
1341  return rv;
1342 }
1343 
1344 /**
1345  * Given an array of <b>n</b> compressed_result_t in <b>results</b>,
1346  * as produced by compress_multiple, store them all into the
1347  * consdiffmgr, and store handles to them in the <b>handles_out</b>
1348  * array.
1349  *
1350  * Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none
1351  * was stored.
1352  */
1353 static cdm_diff_status_t
1354 store_multiple(consensus_cache_entry_handle_t **handles_out,
1355  int n,
1356  const compress_method_t *methods,
1357  const compressed_result_t *results,
1358  const char *description)
1359 {
1360  cdm_diff_status_t status = CDM_DIFF_ERROR;
1362 
1363  int i;
1364  for (i = 0; i < n; ++i) {
1365  compress_method_t method = methods[i];
1366  uint8_t *body_out = results[i].body;
1367  size_t bodylen_out = results[i].bodylen;
1368  config_line_t *labels = results[i].labels;
1369  const char *methodname = compression_method_get_name(method);
1370  if (body_out && bodylen_out && labels) {
1371  /* Success! Store the results */
1372  log_info(LD_DIRSERV, "Adding %s, compressed with %s",
1373  description, methodname);
1374 
1377  labels,
1378  body_out,
1379  bodylen_out);
1380  if (ent == NULL) {
1381  static ratelim_t cant_store_ratelim = RATELIM_INIT(5*60);
1382  log_fn_ratelim(&cant_store_ratelim, LOG_WARN, LD_FS,
1383  "Unable to store object %s compressed with %s.",
1384  description, methodname);
1385  continue;
1386  }
1387 
1388  status = CDM_DIFF_PRESENT;
1389  handles_out[i] = consensus_cache_entry_handle_new(ent);
1391  }
1392  }
1393  return status;
1394 }
1395 
1396 /**
1397  * An object passed to a worker thread that will try to produce a consensus
1398  * diff.
1399  */
1401  /**
1402  * Input: The consensus to compute the diff from. Holds a reference to the
1403  * cache entry, which must not be released until the job is passed back to
1404  * the main thread. The body must be mapped into memory in the main thread.
1405  */
1407  /**
1408  * Input: The consensus to compute the diff to. Holds a reference to the
1409  * cache entry, which must not be released until the job is passed back to
1410  * the main thread. The body must be mapped into memory in the main thread.
1411  */
1413 
1414  /** Output: labels and bodies */
1417 
1418 /** Given a consensus_cache_entry_t, check whether it has a label claiming
1419  * that it was compressed. If so, uncompress its contents into *<b>out</b> and
1420  * set <b>outlen</b> to hold their size, and set *<b>owned_out</b> to a pointer
1421  * that the caller will need to free. If not, just set *<b>out</b> and
1422  * <b>outlen</b> to its extent in memory. Return 0 on success, -1 on failure.
1423  **/
1424 STATIC int
1425 uncompress_or_set_ptr(const char **out, size_t *outlen,
1426  char **owned_out,
1428 {
1429  const uint8_t *body;
1430  size_t bodylen;
1431 
1432  *owned_out = NULL;
1433 
1434  if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0)
1435  return -1;
1436 
1437  const char *lv_compression =
1438  consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
1439  compress_method_t method = NO_METHOD;
1440 
1441  if (lv_compression)
1442  method = compression_method_get_by_name(lv_compression);
1443 
1444  int rv;
1445  if (method == NO_METHOD) {
1446  *out = (const char *)body;
1447  *outlen = bodylen;
1448  rv = 0;
1449  } else {
1450  rv = tor_uncompress(owned_out, outlen, (const char *)body, bodylen,
1451  method, 1, LOG_WARN);
1452  *out = *owned_out;
1453  }
1454  return rv;
1455 }
1456 
1457 /**
1458  * Worker function. This function runs inside a worker thread and receives
1459  * a consensus_diff_worker_job_t as its input.
1460  */
1461 static workqueue_reply_t
1462 consensus_diff_worker_threadfn(void *state_, void *work_)
1463 {
1464  (void)state_;
1465  consensus_diff_worker_job_t *job = work_;
1466  const uint8_t *diff_from, *diff_to;
1467  size_t len_from, len_to;
1468  int r;
1469  /* We need to have the body already mapped into RAM here.
1470  */
1471  r = consensus_cache_entry_get_body(job->diff_from, &diff_from, &len_from);
1472  if (BUG(r < 0))
1473  return WQ_RPL_REPLY; // LCOV_EXCL_LINE
1474  r = consensus_cache_entry_get_body(job->diff_to, &diff_to, &len_to);
1475  if (BUG(r < 0))
1476  return WQ_RPL_REPLY; // LCOV_EXCL_LINE
1477 
1478  const char *lv_to_valid_after =
1479  consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER);
1480  const char *lv_to_fresh_until =
1481  consensus_cache_entry_get_value(job->diff_to, LABEL_FRESH_UNTIL);
1482  const char *lv_to_valid_until =
1483  consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_UNTIL);
1484  const char *lv_to_signatories =
1485  consensus_cache_entry_get_value(job->diff_to, LABEL_SIGNATORIES);
1486  const char *lv_from_valid_after =
1487  consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER);
1488  const char *lv_from_digest =
1490  LABEL_SHA3_DIGEST_AS_SIGNED);
1491  const char *lv_from_flavor =
1492  consensus_cache_entry_get_value(job->diff_from, LABEL_FLAVOR);
1493  const char *lv_to_flavor =
1494  consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
1495  const char *lv_to_digest =
1497  LABEL_SHA3_DIGEST_UNCOMPRESSED);
1498 
1499  if (! lv_from_digest) {
1500  /* This isn't a bug right now, since it can happen if you're migrating
1501  * from an older version of master to a newer one. The older ones didn't
1502  * annotate their stored consensus objects with sha3-digest-as-signed.
1503  */
1504  return WQ_RPL_REPLY; // LCOV_EXCL_LINE
1505  }
1506 
1507  /* All these values are mandatory on the input */
1508  if (BUG(!lv_to_valid_after) ||
1509  BUG(!lv_from_valid_after) ||
1510  BUG(!lv_from_flavor) ||
1511  BUG(!lv_to_flavor)) {
1512  return WQ_RPL_REPLY; // LCOV_EXCL_LINE
1513  }
1514  /* The flavors need to match */
1515  if (BUG(strcmp(lv_from_flavor, lv_to_flavor))) {
1516  return WQ_RPL_REPLY; // LCOV_EXCL_LINE
1517  }
1518 
1519  char *consensus_diff;
1520  {
1521  const char *diff_from_nt = NULL, *diff_to_nt = NULL;
1522  char *owned1 = NULL, *owned2 = NULL;
1523  size_t diff_from_nt_len, diff_to_nt_len;
1524 
1525  if (uncompress_or_set_ptr(&diff_from_nt, &diff_from_nt_len, &owned1,
1526  job->diff_from) < 0) {
1527  return WQ_RPL_REPLY;
1528  }
1529  if (uncompress_or_set_ptr(&diff_to_nt, &diff_to_nt_len, &owned2,
1530  job->diff_to) < 0) {
1531  tor_free(owned1);
1532  return WQ_RPL_REPLY;
1533  }
1534  tor_assert(diff_from_nt);
1535  tor_assert(diff_to_nt);
1536 
1537  // XXXX ugh; this is going to calculate the SHA3 of both its
1538  // XXXX inputs again, even though we already have that. Maybe it's time
1539  // XXXX to change the API here?
1540  consensus_diff = consensus_diff_generate(diff_from_nt,
1541  diff_from_nt_len,
1542  diff_to_nt,
1543  diff_to_nt_len);
1544  tor_free(owned1);
1545  tor_free(owned2);
1546  }
1547  if (!consensus_diff) {
1548  /* Couldn't generate consensus; we'll leave the reply blank. */
1549  return WQ_RPL_REPLY;
1550  }
1551 
1552  /* Compress the results and send the reply */
1553  tor_assert(compress_diffs_with[0] == NO_METHOD);
1554  size_t difflen = strlen(consensus_diff);
1555  job->out[0].body = (uint8_t *) consensus_diff;
1556  job->out[0].bodylen = difflen;
1557 
1558  config_line_t *common_labels = NULL;
1559  if (lv_to_valid_until)
1560  config_line_prepend(&common_labels, LABEL_VALID_UNTIL, lv_to_valid_until);
1561  if (lv_to_fresh_until)
1562  config_line_prepend(&common_labels, LABEL_FRESH_UNTIL, lv_to_fresh_until);
1563  if (lv_to_signatories)
1564  config_line_prepend(&common_labels, LABEL_SIGNATORIES, lv_to_signatories);
1565  cdm_labels_prepend_sha3(&common_labels,
1566  LABEL_SHA3_DIGEST_UNCOMPRESSED,
1567  job->out[0].body,
1568  job->out[0].bodylen);
1569  config_line_prepend(&common_labels, LABEL_FROM_VALID_AFTER,
1570  lv_from_valid_after);
1571  config_line_prepend(&common_labels, LABEL_VALID_AFTER,
1572  lv_to_valid_after);
1573  config_line_prepend(&common_labels, LABEL_FLAVOR, lv_from_flavor);
1574  config_line_prepend(&common_labels, LABEL_FROM_SHA3_DIGEST,
1575  lv_from_digest);
1576  config_line_prepend(&common_labels, LABEL_TARGET_SHA3_DIGEST,
1577  lv_to_digest);
1578  config_line_prepend(&common_labels, LABEL_DOCTYPE,
1579  DOCTYPE_CONSENSUS_DIFF);
1580 
1581  job->out[0].labels = config_lines_dup(common_labels);
1582  cdm_labels_prepend_sha3(&job->out[0].labels,
1583  LABEL_SHA3_DIGEST,
1584  job->out[0].body,
1585  job->out[0].bodylen);
1586 
1587  compress_multiple(job->out+1,
1590  (const uint8_t*)consensus_diff, difflen, common_labels);
1591 
1592  config_free_lines(common_labels);
1593  return WQ_RPL_REPLY;
1594 }
1595 
1596 #define consensus_diff_worker_job_free(job) \
1597  FREE_AND_NULL(consensus_diff_worker_job_t, \
1598  consensus_diff_worker_job_free_, (job))
1599 
1600 /**
1601  * Helper: release all storage held in <b>job</b>.
1602  */
1603 static void
1605 {
1606  if (!job)
1607  return;
1608  unsigned u;
1609  for (u = 0; u < n_diff_compression_methods(); ++u) {
1610  config_free_lines(job->out[u].labels);
1611  tor_free(job->out[u].body);
1612  }
1615  tor_free(job);
1616 }
1617 
1618 /**
1619  * Worker function: This function runs in the main thread, and receives
1620  * a consensus_diff_worker_job_t that the worker thread has already
1621  * processed.
1622  */
1623 static void
1625 {
1627  tor_assert(work_);
1628 
1629  consensus_diff_worker_job_t *job = work_;
1630 
1631  const char *lv_from_digest =
1633  LABEL_SHA3_DIGEST_AS_SIGNED);
1634  const char *lv_to_digest =
1636  LABEL_SHA3_DIGEST_UNCOMPRESSED);
1637  const char *lv_flavor =
1638  consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
1639  if (BUG(lv_from_digest == NULL))
1640  lv_from_digest = "???"; // LCOV_EXCL_LINE
1641  if (BUG(lv_to_digest == NULL))
1642  lv_to_digest = "???"; // LCOV_EXCL_LINE
1643 
1644  uint8_t from_sha3[DIGEST256_LEN];
1645  uint8_t to_sha3[DIGEST256_LEN];
1646  int flav = -1;
1647  int cache = 1;
1648  if (BUG(cdm_entry_get_sha3_value(from_sha3, job->diff_from,
1649  LABEL_SHA3_DIGEST_AS_SIGNED) < 0))
1650  cache = 0;
1651  if (BUG(cdm_entry_get_sha3_value(to_sha3, job->diff_to,
1652  LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
1653  cache = 0;
1654  if (BUG(lv_flavor == NULL)) {
1655  cache = 0;
1656  } else if ((flav = networkstatus_parse_flavor_name(lv_flavor)) < 0) {
1657  cache = 0;
1658  }
1659 
1660  consensus_cache_entry_handle_t *handles[ARRAY_LENGTH(compress_diffs_with)];
1661  memset(handles, 0, sizeof(handles));
1662 
1663  char description[128];
1664  tor_snprintf(description, sizeof(description),
1665  "consensus diff from %s to %s",
1666  lv_from_digest, lv_to_digest);
1667 
1668  int status = store_multiple(handles,
1671  job->out,
1672  description);
1673 
1674  if (status != CDM_DIFF_PRESENT) {
1675  /* Failure! Nothing to do but complain */
1676  log_warn(LD_DIRSERV,
1677  "Worker was unable to compute consensus diff "
1678  "from %s to %s", lv_from_digest, lv_to_digest);
1679  /* Cache this error so we don't try to compute this one again. */
1680  status = CDM_DIFF_ERROR;
1681  }
1682 
1683  unsigned u;
1684  for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
1686  if (cache) {
1687  consensus_cache_entry_handle_t *h = handles[u];
1688  int this_status = status;
1689  if (h == NULL) {
1690  this_status = CDM_DIFF_ERROR;
1691  }
1692  tor_assert_nonfatal(h != NULL || this_status == CDM_DIFF_ERROR);
1693  cdm_diff_ht_set_status(flav, from_sha3, to_sha3, method, this_status, h);
1694  } else {
1695  consensus_cache_entry_handle_free(handles[u]);
1696  }
1697  }
1698 
1699  consensus_diff_worker_job_free(job);
1700 }
1701 
1702 /**
1703  * Queue the job of computing the diff from <b>diff_from</b> to <b>diff_to</b>
1704  * in a worker thread.
1705  */
1706 static int
1708  consensus_cache_entry_t *diff_to)
1709 {
1711 
1712  consensus_cache_entry_incref(diff_from);
1714 
1715  consensus_diff_worker_job_t *job = tor_malloc_zero(sizeof(*job));
1716  job->diff_from = diff_from;
1717  job->diff_to = diff_to;
1718 
1719  /* Make sure body is mapped. */
1720  const uint8_t *body;
1721  size_t bodylen;
1722  int r1 = consensus_cache_entry_get_body(diff_from, &body, &bodylen);
1723  int r2 = consensus_cache_entry_get_body(diff_to, &body, &bodylen);
1724  if (r1 < 0 || r2 < 0)
1725  goto err;
1726 
1727  workqueue_entry_t *work;
1728  work = cpuworker_queue_work(WQ_PRI_LOW,
1731  job);
1732  if (!work)
1733  goto err;
1734 
1735  return 0;
1736  err:
1737  consensus_diff_worker_job_free(job); // includes decrefs.
1738  return -1;
1739 }
1740 
1741 /**
1742  * Holds requests and replies for consensus_compress_workers.
1743  */
1745  char *consensus;
1746  size_t consensus_len;
1747  consensus_flavor_t flavor;
1748  config_line_t *labels_in;
1751 
1752 #define consensus_compress_worker_job_free(job) \
1753  FREE_AND_NULL(consensus_compress_worker_job_t, \
1754  consensus_compress_worker_job_free_, (job))
1755 
1756 /**
1757  * Free all resources held in <b>job</b>
1758  */
1759 static void
1761 {
1762  if (!job)
1763  return;
1764  tor_free(job->consensus);
1765  config_free_lines(job->labels_in);
1766  unsigned u;
1767  for (u = 0; u < n_consensus_compression_methods(); ++u) {
1768  config_free_lines(job->out[u].labels);
1769  tor_free(job->out[u].body);
1770  }
1771  tor_free(job);
1772 }
1773 /**
1774  * Worker function. This function runs inside a worker thread and receives
1775  * a consensus_compress_worker_job_t as its input.
1776  */
1777 static workqueue_reply_t
1778 consensus_compress_worker_threadfn(void *state_, void *work_)
1779 {
1780  (void)state_;
1781  consensus_compress_worker_job_t *job = work_;
1782  consensus_flavor_t flavor = job->flavor;
1783  const char *consensus = job->consensus;
1784  size_t bodylen = job->consensus_len;
1785 
1786  config_line_t *labels = config_lines_dup(job->labels_in);
1787  const char *flavname = networkstatus_get_flavor_name(flavor);
1788 
1789  cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED,
1790  (const uint8_t *)consensus, bodylen);
1791  {
1792  const char *start, *end;
1793  if (router_get_networkstatus_v3_signed_boundaries(consensus, bodylen,
1794  &start, &end) < 0) {
1795  start = consensus;
1796  end = consensus+bodylen;
1797  }
1798  cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED,
1799  (const uint8_t *)start,
1800  end - start);
1801  }
1802  config_line_prepend(&labels, LABEL_FLAVOR, flavname);
1803  config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
1804 
1805  compress_multiple(job->out,
1808  (const uint8_t*)consensus, bodylen, labels);
1809  config_free_lines(labels);
1810  return WQ_RPL_REPLY;
1811 }
1812 
1813 /**
1814  * Worker function: This function runs in the main thread, and receives
1815  * a consensus_diff_compress_job_t that the worker thread has already
1816  * processed.
1817  */
1818 static void
1820 {
1821  consensus_compress_worker_job_t *job = work_;
1822 
1823  consensus_cache_entry_handle_t *handles[
1825  memset(handles, 0, sizeof(handles));
1826 
1827  store_multiple(handles,
1830  job->out,
1831  "consensus");
1833 
1834  unsigned u;
1835  consensus_flavor_t f = job->flavor;
1836  tor_assert((int)f < N_CONSENSUS_FLAVORS);
1837  for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
1838  if (handles[u] == NULL)
1839  continue;
1840  consensus_cache_entry_handle_free(latest_consensus[f][u]);
1841  latest_consensus[f][u] = handles[u];
1842  }
1843 
1844  consensus_compress_worker_job_free(job);
1845 }
1846 
1847 /**
1848  * If true, we compress in worker threads.
1849  */
1850 static int background_compression = 0;
1851 
1852 /**
1853  * Queue a job to compress <b>consensus</b> and store its compressed
1854  * text in the cache.
1855  */
1856 static int
1857 consensus_queue_compression_work(const char *consensus,
1858  size_t consensus_len,
1859  const networkstatus_t *as_parsed)
1860 {
1861  tor_assert(consensus);
1862  tor_assert(as_parsed);
1863 
1864  consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job));
1865  job->consensus = tor_memdup_nulterm(consensus, consensus_len);
1866  job->consensus_len = strlen(job->consensus);
1867  job->flavor = as_parsed->flavor;
1868 
1869  char va_str[ISO_TIME_LEN+1];
1870  char vu_str[ISO_TIME_LEN+1];
1871  char fu_str[ISO_TIME_LEN+1];
1872  format_iso_time_nospace(va_str, as_parsed->valid_after);
1873  format_iso_time_nospace(fu_str, as_parsed->fresh_until);
1874  format_iso_time_nospace(vu_str, as_parsed->valid_until);
1875  config_line_append(&job->labels_in, LABEL_VALID_AFTER, va_str);
1876  config_line_append(&job->labels_in, LABEL_FRESH_UNTIL, fu_str);
1877  config_line_append(&job->labels_in, LABEL_VALID_UNTIL, vu_str);
1878  if (as_parsed->voters) {
1879  smartlist_t *hexvoters = smartlist_new();
1880  SMARTLIST_FOREACH_BEGIN(as_parsed->voters,
1882  if (smartlist_len(vi->sigs) == 0)
1883  continue; // didn't sign.
1884  char d[HEX_DIGEST_LEN+1];
1885  base16_encode(d, sizeof(d), vi->identity_digest, DIGEST_LEN);
1886  smartlist_add_strdup(hexvoters, d);
1887  } SMARTLIST_FOREACH_END(vi);
1888  char *signers = smartlist_join_strings(hexvoters, ",", 0, NULL);
1889  config_line_prepend(&job->labels_in, LABEL_SIGNATORIES, signers);
1890  tor_free(signers);
1891  SMARTLIST_FOREACH(hexvoters, char *, cp, tor_free(cp));
1892  smartlist_free(hexvoters);
1893  }
1894 
1895  if (background_compression) {
1896  workqueue_entry_t *work;
1897  work = cpuworker_queue_work(WQ_PRI_LOW,
1900  job);
1901  if (!work) {
1902  consensus_compress_worker_job_free(job);
1903  return -1;
1904  }
1905 
1906  return 0;
1907  } else {
1910  return 0;
1911  }
1912 }
1913 
1914 /**
1915  * Tell the consdiffmgr backend to compress consensuses in worker threads.
1916  */
1917 void
1919 {
1920  // This isn't the default behavior because it would break unit tests.
1922 }
1923 
1924 /** Read the set of voters from the cached object <b>ent</b> into
1925  * <b>out</b>, as a list of hex-encoded digests. Return 0 on success,
1926  * -1 if no signatories were recorded. */
1927 int
1929  smartlist_t *out)
1930 {
1931  tor_assert(ent);
1932  tor_assert(out);
1933  const char *s;
1934  s = consensus_cache_entry_get_value(ent, LABEL_SIGNATORIES);
1935  if (s == NULL)
1936  return -1;
1937  smartlist_split_string(out, s, ",", SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE, 0);
1938  return 0;
1939 }
1940 
1941 /** Read the fresh-until time of cached object <b>ent</b> into *<b>out</b>
1942  * and return 0, or return -1 if no such time was recorded. */
1943 int
1945  time_t *out)
1946 {
1947  tor_assert(ent);
1948  tor_assert(out);
1949  const char *s;
1950  s = consensus_cache_entry_get_value(ent, LABEL_FRESH_UNTIL);
1951  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
1952  return -1;
1953  else
1954  return 0;
1955 }
1956 
1957 /** Read the valid until timestamp from the cached object <b>ent</b> into
1958  * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
1959 int
1961  time_t *out)
1962 {
1963  tor_assert(ent);
1964  tor_assert(out);
1965 
1966  const char *s;
1967  s = consensus_cache_entry_get_value(ent, LABEL_VALID_UNTIL);
1968  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
1969  return -1;
1970  else
1971  return 0;
1972 }
1973 
1974 /** Read the valid after timestamp from the cached object <b>ent</b> into
1975  * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
1976 int
1978  time_t *out)
1979 {
1980  tor_assert(ent);
1981  tor_assert(out);
1982 
1983  const char *s;
1984  s = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
1985 
1986  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
1987  return -1;
1988  else
1989  return 0;
1990 }
time_t approx_time(void)
Definition: approx_time.c:32
int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:506
void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:478
#define ARRAY_LENGTH(x)
mainloop_event_t * mainloop_event_postloop_new(void(*cb)(mainloop_event_t *, void *), void *userdata)
void mainloop_event_activate(mainloop_event_t *event)
Header for compat_libevent.c.
int in_main_thread(void)
compress_method_t compression_method_get_by_name(const char *name)
Definition: compress.c:403
const char * compression_method_get_name(compress_method_t method)
Definition: compress.c:364
int tor_compress(char **out, size_t *out_len, const char *in, size_t in_len, compress_method_t method)
Definition: compress.c:242
int tor_uncompress(char **out, size_t *out_len, const char *in, size_t in_len, compress_method_t method, int complete_only, int protocol_warn_level)
Definition: compress.c:268
Headers for compress.c.
compress_method_t
Definition: compress.h:21
const or_options_t * get_options(void)
Definition: config.c:919
Header file for config.c.
void config_line_prepend(config_line_t **lst, const char *key, const char *val)
Definition: confline.c:53
config_line_t * config_lines_dup(const config_line_t *inp)
Definition: confline.c:226
void config_line_append(config_line_t **lst, const char *key, const char *val)
Definition: confline.c:32
Header for confline.c.
void consensus_cache_entry_incref(consensus_cache_entry_t *ent)
Definition: conscache.c:355
consensus_cache_entry_t * consensus_cache_add(consensus_cache_t *cache, const config_line_t *labels, const uint8_t *data, size_t datalen)
Definition: conscache.c:221
void consensus_cache_find_all(smartlist_t *out, consensus_cache_t *cache, const char *key, const char *value)
Definition: conscache.c:277
void consensus_cache_delete_pending(consensus_cache_t *cache, int force)
Definition: conscache.c:509
void consensus_cache_entry_mark_for_aggressive_release(consensus_cache_entry_t *ent)
Definition: conscache.c:423
int consensus_cache_get_n_filenames_available(consensus_cache_t *cache)
Definition: conscache.c:489
const char * consensus_cache_entry_get_value(const consensus_cache_entry_t *ent, const char *key)
Definition: conscache.c:329
int consensus_cache_may_overallocate(consensus_cache_t *cache)
Definition: conscache.c:125
int consensus_cache_register_with_sandbox(consensus_cache_t *cache, struct sandbox_cfg_elem_t **cfg)
Definition: conscache.c:149
void consensus_cache_filter_list(smartlist_t *lst, const char *key, const char *value)
Definition: conscache.c:305
void consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent)
Definition: conscache.c:413
void consensus_cache_entry_decref(consensus_cache_entry_t *ent)
Definition: conscache.c:370
int consensus_cache_entry_get_body(const consensus_cache_entry_t *ent, const uint8_t **body_out, size_t *sz_out)
Definition: conscache.c:437
consensus_cache_t * consensus_cache_open(const char *subdir, int max_entries)
Definition: conscache.c:85
Header for conscache.c.
char * consensus_diff_generate(const char *cons1, size_t cons1len, const char *cons2, size_t cons2len)
Definition: consdiff.c:1341
Header for consdiff.c.
int consensus_cache_entry_get_valid_until(const consensus_cache_entry_t *ent, time_t *out)
Definition: consdiffmgr.c:1960
static void consdiffmgr_consensus_load(void)
Definition: consdiffmgr.c:1038
static cdm_diff_t * cdm_diff_new(consensus_flavor_t flav, const uint8_t *from_sha3, const uint8_t *target_sha3, compress_method_t method)
Definition: consdiffmgr.c:251
static cdm_diff_status_t store_multiple(consensus_cache_entry_handle_t **handles_out, int n, const compress_method_t *methods, const compressed_result_t *results, const char *description)
Definition: consdiffmgr.c:1354
#define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD
Definition: consdiffmgr.c:145
int consensus_cache_entry_get_fresh_until(const consensus_cache_entry_t *ent, time_t *out)
Definition: consdiffmgr.c:1944
consdiff_status_t consdiffmgr_find_diff_from(consensus_cache_entry_t **entry_out, consensus_flavor_t flavor, int digest_type, const uint8_t *digest, size_t digestlen, compress_method_t method)
Definition: consdiffmgr.c:660
void consdiffmgr_enable_background_compression(void)
Definition: consdiffmgr.c:1918
void consdiffmgr_configure(const consdiff_cfg_t *cfg)
Definition: consdiffmgr.c:844
static HT_HEAD(cdm_diff_ht, cdm_diff_t)
Definition: consdiffmgr.c:178
int consdiffmgr_validate(void)
Definition: consdiffmgr.c:868
static void cdm_cache_init(void)
Definition: consdiffmgr.c:386
static void consdiffmgr_diffs_load(void)
Definition: consdiffmgr.c:1079
static void cdm_diff_ht_set_status(consensus_flavor_t flav, const uint8_t *from_sha3, const uint8_t *to_sha3, compress_method_t method, int status, consensus_cache_entry_handle_t *handle)
Definition: consdiffmgr.c:307
STATIC consensus_cache_t * cdm_cache_get(void)
Definition: consdiffmgr.c:411
static int consensus_compression_method_pos(compress_method_t method)
Definition: consdiffmgr.c:612
static void consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg)
Definition: consdiffmgr.c:1145
STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out, consensus_cache_entry_t *ent, const char *label)
Definition: consdiffmgr.c:445
static const compress_method_t compress_consensus_with[]
Definition: consdiffmgr.c:124
static const compress_method_t compress_diffs_with[]
Definition: consdiffmgr.c:97
static void cdm_diff_ht_purge(consensus_flavor_t flav, const uint8_t *unless_target_sha3_matches)
Definition: consdiffmgr.c:351
static workqueue_reply_t consensus_compress_worker_threadfn(void *state_, void *work_)
Definition: consdiffmgr.c:1778
static int cdm_cache_dirty
Definition: consdiffmgr.c:81
int consdiffmgr_cleanup(void)
Definition: consdiffmgr.c:722
#define LABEL_DOCTYPE
Definition: consdiffmgr.c:38
static void consdiffmgr_rescan_flavor_(consensus_flavor_t flavor)
Definition: consdiffmgr.c:920
STATIC int uncompress_or_set_ptr(const char **out, size_t *outlen, char **owned_out, consensus_cache_entry_t *ent)
Definition: consdiffmgr.c:1425
STATIC unsigned n_consensus_compression_methods(void)
Definition: consdiffmgr.c:136
static consensus_cache_t * cons_diff_cache
Definition: consdiffmgr.c:76
static int cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2)
Definition: consdiffmgr.c:224
static void consdiffmgr_set_cache_flags(void)
Definition: consdiffmgr.c:1250
int consensus_cache_entry_get_voter_id_digests(const consensus_cache_entry_t *ent, smartlist_t *out)
Definition: consdiffmgr.c:1928
static int compare_by_valid_after_(const void **a, const void **b)
Definition: consdiffmgr.c:584
static unsigned cdm_diff_hash(const cdm_diff_t *diff)
Definition: consdiffmgr.c:214
static int background_compression
Definition: consdiffmgr.c:1850
static int compare_by_staleness_(const void **a, const void **b)
Definition: consdiffmgr.c:1168
consdiff_status_t consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out, consensus_flavor_t flavor, compress_method_t method)
Definition: consdiffmgr.c:629
static consensus_cache_entry_t * sort_and_find_most_recent(smartlist_t *lst)
Definition: consdiffmgr.c:599
static void consensus_compress_worker_job_free_(consensus_compress_worker_job_t *job)
Definition: consdiffmgr.c:1760
int consdiffmgr_add_consensus(const char *consensus, size_t consensus_len, const networkstatus_t *as_parsed)
Definition: consdiffmgr.c:549
static int consdiffmgr_ensure_space_for_files(int n)
Definition: consdiffmgr.c:1195
STATIC consensus_cache_entry_t * cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after)
Definition: consdiffmgr.c:469
static mainloop_event_t * consdiffmgr_rescan_ev
Definition: consdiffmgr.c:111
STATIC unsigned n_diff_compression_methods(void)
Definition: consdiffmgr.c:118
static int cdm_cache_loaded
Definition: consdiffmgr.c:85
int consensus_cache_entry_get_valid_after(const consensus_cache_entry_t *ent, time_t *out)
Definition: consdiffmgr.c:1977
void consdiffmgr_rescan(void)
Definition: consdiffmgr.c:1121
static int consensus_queue_compression_work(const char *consensus, size_t consensus_len, const networkstatus_t *as_parsed)
Definition: consdiffmgr.c:1857
static void consensus_diff_worker_job_free_(consensus_diff_worker_job_t *job)
Definition: consdiffmgr.c:1604
void consdiffmgr_free_all(void)
Definition: consdiffmgr.c:1267
static void mark_cdm_cache_dirty(void)
Definition: consdiffmgr.c:1154
static void consensus_diff_worker_replyfn(void *work_)
Definition: consdiffmgr.c:1624
cdm_diff_status_t
Definition: consdiffmgr.c:90
static int compress_multiple(compressed_result_t *results_out, int n_methods, const compress_method_t *methods, const uint8_t *input, size_t len, const config_line_t *labels_in)
Definition: consdiffmgr.c:1315
static int cdm_diff_ht_check_and_note_pending(consensus_flavor_t flav, const uint8_t *from_sha3, const uint8_t *target_sha3)
Definition: consdiffmgr.c:274
static int32_t get_max_age_to_cache(void)
Definition: consdiffmgr.c:498
static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, consensus_cache_entry_t *diff_to)
Definition: consdiffmgr.c:1707
static workqueue_reply_t consensus_diff_worker_threadfn(void *state_, void *work_)
Definition: consdiffmgr.c:1462
int consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem_t **cfg)
Definition: consdiffmgr.c:857
static consensus_cache_entry_handle_t * latest_consensus[N_CONSENSUS_FLAVORS][ARRAY_LENGTH(compress_consensus_with)]
Definition: consdiffmgr.c:151
static void cdm_diff_free_(cdm_diff_t *diff)
Definition: consdiffmgr.c:240
static void consensus_compress_worker_replyfn(void *work_)
Definition: consdiffmgr.c:1819
static void cdm_labels_prepend_sha3(config_line_t **labels, const char *label, const uint8_t *body, size_t bodylen)
Definition: consdiffmgr.c:425
Header for consdiffmgr.c.
consdiff_status_t
Definition: consdiffmgr.h:17
workqueue_entry_t * cpuworker_queue_work(workqueue_priority_t priority, workqueue_reply_t(*fn)(void *, void *), void(*reply_fn)(void *), void *arg)
Definition: cpuworker.c:496
Header file for cpuworker.c.
#define HEX_DIGEST256_LEN
Definition: crypto_digest.h:37
#define HEX_DIGEST_LEN
Definition: crypto_digest.h:35
int crypto_digest256(char *digest, const char *m, size_t len, digest_algorithm_t algorithm)
#define fast_memeq(a, b, c)
Definition: di_ops.h:35
#define fast_memneq(a, b, c)
Definition: di_ops.h:42
#define DIGEST_LEN
Definition: digest_sizes.h:20
#define DIGEST256_LEN
Definition: digest_sizes.h:23
const char * escaped(const char *s)
Definition: escape.c:126
HT_PROTOTYPE(hs_circuitmap_ht, circuit_t, hs_circuitmap_node, hs_circuit_hash_token, hs_circuits_have_same_token)
#define log_fn_ratelim(ratelim, severity, domain, args,...)
Definition: log.h:288
#define LD_DIRSERV
Definition: log.h:90
#define LD_FS
Definition: log.h:70
#define LOG_WARN
Definition: log.h:53
void tor_free_(void *mem)
Definition: malloc.c:227
#define tor_free(p)
Definition: malloc.h:52
int networkstatus_parse_flavor_name(const char *flavname)
const char * networkstatus_get_flavor_name(consensus_flavor_t flav)
int32_t networkstatus_get_param(const networkstatus_t *ns, const char *param_name, int32_t default_val, int32_t min_val, int32_t max_val)
Header file for networkstatus.c.
Networkstatus consensus/vote structure.
Single consensus voter structure.
Header file for ns_parse.c.
int router_get_networkstatus_v3_signed_boundaries(const char *s, size_t len, const char **start_out, const char **end_out)
Definition: ns_parse.c:163
Master header file for Tor-specific functionality.
consensus_flavor_t
Definition: or.h:761
#define N_CONSENSUS_FLAVORS
Definition: or.h:767
int tor_snprintf(char *str, size_t size, const char *format,...)
Definition: printf.c:27
void smartlist_reverse(smartlist_t *sl)
Definition: smartlist.c:59
char * smartlist_join_strings(smartlist_t *sl, const char *join, int terminate, size_t *len_out)
Definition: smartlist.c:279
void smartlist_sort(smartlist_t *sl, int(*compare)(const void **a, const void **b))
Definition: smartlist.c:334
smartlist_t * smartlist_new(void)
void smartlist_add_strdup(struct smartlist_t *sl, const char *string)
void smartlist_add(smartlist_t *sl, void *element)
void smartlist_clear(smartlist_t *sl)
void smartlist_del(smartlist_t *sl, int idx)
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
#define SMARTLIST_FOREACH(sl, type, var, cmd)
int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep, int flags, int max)
Definition: conscache.c:32
compressed_result_t out[ARRAY_LENGTH(compress_diffs_with)]
Definition: consdiffmgr.c:1415
consensus_cache_entry_t * diff_from
Definition: consdiffmgr.c:1406
consensus_cache_entry_t * diff_to
Definition: consdiffmgr.c:1412
smartlist_t * voters
consensus_flavor_t flavor
networkstatus_type_t type
int MaxConsensusAgeForDiffs
Definition: workqueue.c:95
#define STATIC
Definition: testsupport.h:32
void format_iso_time_nospace(char *buf, time_t t)
Definition: time_fmt.c:313
int parse_iso_time_nospace(const char *cp, time_t *t)
Definition: time_fmt.c:401
#define tor_assert(expr)
Definition: util_bug.h:102
int strcmp_opt(const char *s1, const char *s2)
Definition: util_string.c:197
Header for workqueue.c.
workqueue_reply_t
Definition: workqueue.h:24