tor  0.4.2.0-alpha-dev
consdiffmgr.c
1 /* Copyright (c) 2017-2019, The Tor Project, Inc. */
2 /* See LICENSE for licensing information */
3 
14 #define CONSDIFFMGR_PRIVATE
15 
16 #include "core/or/or.h"
17 #include "app/config/config.h"
18 #include "feature/dircache/conscache.h"
19 #include "feature/dircommon/consdiff.h"
20 #include "feature/dircache/consdiffmgr.h"
25 #include "lib/evloop/workqueue.h"
26 #include "lib/compress/compress.h"
27 #include "lib/encoding/confline.h"
28 
29 #include "feature/nodelist/networkstatus_st.h"
30 #include "feature/nodelist/networkstatus_voter_info_st.h"
31 
37 /* One of DOCTYPE_CONSENSUS or DOCTYPE_CONSENSUS_DIFF */
38 #define LABEL_DOCTYPE "document-type"
39 /* The valid-after time for a consensus (or for the target consensus of a
40  * diff), encoded as ISO UTC. */
41 #define LABEL_VALID_AFTER "consensus-valid-after"
42 /* The fresh-until time for a consensus (or for the target consensus of a
43  * diff), encoded as ISO UTC. */
44 #define LABEL_FRESH_UNTIL "consensus-fresh-until"
45 /* The valid-until time for a consensus (or for the target consensus of a
46  * diff), encoded as ISO UTC. */
47 #define LABEL_VALID_UNTIL "consensus-valid-until"
48 /* Comma-separated list of hex-encoded identity digests for the voting
49  * authorities. */
50 #define LABEL_SIGNATORIES "consensus-signatories"
51 /* A hex encoded SHA3 digest of the object, as compressed (if any) */
52 #define LABEL_SHA3_DIGEST "sha3-digest"
53 /* A hex encoded SHA3 digest of the object before compression. */
54 #define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed"
55 /* A hex encoded SHA3 digest-as-signed of a consensus */
56 #define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed"
57 /* The flavor of the consensus or consensuses diff */
58 #define LABEL_FLAVOR "consensus-flavor"
59 /* Diff only: the SHA3 digest-as-signed of the source consensus. */
60 #define LABEL_FROM_SHA3_DIGEST "from-sha3-digest"
61 /* Diff only: the SHA3 digest-in-full of the target consensus. */
62 #define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest"
63 /* Diff only: the valid-after date of the source consensus. */
64 #define LABEL_FROM_VALID_AFTER "from-valid-after"
65 /* What kind of compression was used? */
66 #define LABEL_COMPRESSION_TYPE "compression"
67 
69 #define DOCTYPE_CONSENSUS "consensus"
70 #define DOCTYPE_CONSENSUS_DIFF "consensus-diff"
71 
76 static consensus_cache_t *cons_diff_cache = NULL;
81 static int cdm_cache_dirty = 0;
85 static int cdm_cache_loaded = 0;
86 
90 typedef enum cdm_diff_status_t {
91  CDM_DIFF_PRESENT=1,
92  CDM_DIFF_IN_PROGRESS=2,
93  CDM_DIFF_ERROR=3,
94 } cdm_diff_status_t;
95 
97 static const compress_method_t compress_diffs_with[] = {
98  NO_METHOD,
99  GZIP_METHOD,
100 #ifdef HAVE_LZMA
101  LZMA_METHOD,
102 #endif
103 #ifdef HAVE_ZSTD
104  ZSTD_METHOD,
105 #endif
106 };
107 
111 static mainloop_event_t *consdiffmgr_rescan_ev = NULL;
112 
113 static void consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg);
114 static void mark_cdm_cache_dirty(void);
115 
117 STATIC unsigned
118 n_diff_compression_methods(void)
119 {
120  return ARRAY_LENGTH(compress_diffs_with);
121 }
122 
124 static const compress_method_t compress_consensus_with[] = {
125  ZLIB_METHOD,
126 #ifdef HAVE_LZMA
127  LZMA_METHOD,
128 #endif
129 #ifdef HAVE_ZSTD
130  ZSTD_METHOD,
131 #endif
132 };
133 
135 STATIC unsigned
136 n_consensus_compression_methods(void)
137 {
138  return ARRAY_LENGTH(compress_consensus_with);
139 }
140 
145 #define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD
146 
149 static consensus_cache_entry_handle_t *
150  latest_consensus[N_CONSENSUS_FLAVORS]
151  [ARRAY_LENGTH(compress_consensus_with)];
152 
155 typedef struct cdm_diff_t {
156  HT_ENTRY(cdm_diff_t) node;
157 
159  consensus_flavor_t flavor;
162  uint8_t from_sha3[DIGEST256_LEN];
164  compress_method_t compress_method;
165 
168  cdm_diff_status_t cdm_diff_status;
170  uint8_t target_sha3[DIGEST256_LEN];
171 
174  consensus_cache_entry_handle_t *entry;
175 } cdm_diff_t;
176 
178 static HT_HEAD(cdm_diff_ht, cdm_diff_t) cdm_diff_ht = HT_INITIALIZER();
179 
183 static consdiff_cfg_t consdiff_cfg = {
184  // XXXX I'd like to make this number bigger, but it interferes with the
185  // XXXX seccomp2 syscall filter, which tops out at BPF_MAXINS (4096)
186  // XXXX rules.
187  /* .cache_max_num = */ 128
188 };
189 
190 static int consdiffmgr_ensure_space_for_files(int n);
191 static int consensus_queue_compression_work(const char *consensus,
192  size_t consensus_len,
193  const networkstatus_t *as_parsed);
194 static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
195  consensus_cache_entry_t *diff_to);
196 static void consdiffmgr_set_cache_flags(void);
197 
198 /* =====
199  * Hashtable setup
200  * ===== */
201 
203 static unsigned
204 cdm_diff_hash(const cdm_diff_t *diff)
205 {
206  uint8_t tmp[DIGEST256_LEN + 2];
207  memcpy(tmp, diff->from_sha3, DIGEST256_LEN);
208  tmp[DIGEST256_LEN] = (uint8_t) diff->flavor;
209  tmp[DIGEST256_LEN+1] = (uint8_t) diff->compress_method;
210  return (unsigned) siphash24g(tmp, sizeof(tmp));
211 }
213 static int
214 cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2)
215 {
216  return fast_memeq(diff1->from_sha3, diff2->from_sha3, DIGEST256_LEN) &&
217  diff1->flavor == diff2->flavor &&
218  diff1->compress_method == diff2->compress_method;
219 }
220 
221 HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq)
222 HT_GENERATE2(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq,
223  0.6, tor_reallocarray, tor_free_)
224 
225 #define cdm_diff_free(diff) \
226  FREE_AND_NULL(cdm_diff_t, cdm_diff_free_, (diff))
227 
229 static void
230 cdm_diff_free_(cdm_diff_t *diff)
231 {
232  if (!diff)
233  return;
234  consensus_cache_entry_handle_free(diff->entry);
235  tor_free(diff);
236 }
237 
240 static cdm_diff_t *
241 cdm_diff_new(consensus_flavor_t flav,
242  const uint8_t *from_sha3,
243  const uint8_t *target_sha3,
244  compress_method_t method)
245 {
246  cdm_diff_t *ent;
247  ent = tor_malloc_zero(sizeof(cdm_diff_t));
248  ent->flavor = flav;
249  memcpy(ent->from_sha3, from_sha3, DIGEST256_LEN);
250  memcpy(ent->target_sha3, target_sha3, DIGEST256_LEN);
251  ent->compress_method = method;
252  return ent;
253 }
254 
263 static int
264 cdm_diff_ht_check_and_note_pending(consensus_flavor_t flav,
265  const uint8_t *from_sha3,
266  const uint8_t *target_sha3)
267 {
268  struct cdm_diff_t search, *ent;
269  unsigned u;
270  int result = 0;
271  for (u = 0; u < n_diff_compression_methods(); ++u) {
272  compress_method_t method = compress_diffs_with[u];
273  memset(&search, 0, sizeof(cdm_diff_t));
274  search.flavor = flav;
275  search.compress_method = method;
276  memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
277  ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
278  if (ent) {
279  tor_assert_nonfatal(ent->cdm_diff_status != CDM_DIFF_PRESENT);
280  result = 1;
281  continue;
282  }
283  ent = cdm_diff_new(flav, from_sha3, target_sha3, method);
284  ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
285  HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
286  }
287  return result;
288 }
289 
296 static void
297 cdm_diff_ht_set_status(consensus_flavor_t flav,
298  const uint8_t *from_sha3,
299  const uint8_t *to_sha3,
300  compress_method_t method,
301  int status,
302  consensus_cache_entry_handle_t *handle)
303 {
304  if (handle == NULL) {
305  tor_assert_nonfatal(status != CDM_DIFF_PRESENT);
306  }
307 
308  struct cdm_diff_t search, *ent;
309  memset(&search, 0, sizeof(cdm_diff_t));
310  search.flavor = flav;
311  search.compress_method = method,
312  memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
313  ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
314  if (!ent) {
315  ent = cdm_diff_new(flav, from_sha3, to_sha3, method);
316  ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
317  HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
318  } else if (fast_memneq(ent->target_sha3, to_sha3, DIGEST256_LEN)) {
319  // This can happen under certain really pathological conditions
320  // if we decide we don't care about a diff before it is actually
321  // done computing.
322  return;
323  }
324 
325  tor_assert_nonfatal(ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS);
326 
327  ent->cdm_diff_status = status;
328  consensus_cache_entry_handle_free(ent->entry);
329  ent->entry = handle;
330 }
331 
340 static void
341 cdm_diff_ht_purge(consensus_flavor_t flav,
342  const uint8_t *unless_target_sha3_matches)
343 {
344  cdm_diff_t **diff, **next;
345  for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
346  cdm_diff_t *this = *diff;
347 
348  if ((*diff)->cdm_diff_status == CDM_DIFF_PRESENT &&
349  flav == (*diff)->flavor) {
350 
351  if (BUG((*diff)->entry == NULL) ||
352  consensus_cache_entry_handle_get((*diff)->entry) == NULL) {
353  /* the underlying entry has gone away; drop this. */
354  next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
355  cdm_diff_free(this);
356  continue;
357  }
358 
359  if (unless_target_sha3_matches &&
360  fast_memneq(unless_target_sha3_matches, (*diff)->target_sha3,
361  DIGEST256_LEN)) {
362  /* target hash doesn't match; drop this. */
363  next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
364  cdm_diff_free(this);
365  continue;
366  }
367  }
368  next = HT_NEXT(cdm_diff_ht, &cdm_diff_ht, diff);
369  }
370 }
371 
375 static void
376 cdm_cache_init(void)
377 {
378  unsigned n_entries = consdiff_cfg.cache_max_num * 2;
379 
380  tor_assert(cons_diff_cache == NULL);
381  cons_diff_cache = consensus_cache_open("diff-cache", n_entries);
382  if (cons_diff_cache == NULL) {
383  // LCOV_EXCL_START
384  log_err(LD_FS, "Error: Couldn't open storage for consensus diffs.");
385  tor_assert_unreached();
386  // LCOV_EXCL_STOP
387  } else {
388  consdiffmgr_set_cache_flags();
389  }
390  consdiffmgr_rescan_ev =
391  mainloop_event_postloop_new(consdiffmgr_rescan_cb, NULL);
392  mark_cdm_cache_dirty();
393  cdm_cache_loaded = 0;
394 }
395 
400 STATIC consensus_cache_t *
401 cdm_cache_get(void)
402 {
403  if (PREDICT_UNLIKELY(cons_diff_cache == NULL)) {
404  cdm_cache_init();
405  }
406  return cons_diff_cache;
407 }
408 
414 static void
415 cdm_labels_prepend_sha3(config_line_t **labels,
416  const char *label,
417  const uint8_t *body,
418  size_t bodylen)
419 {
420  uint8_t sha3_digest[DIGEST256_LEN];
421  char hexdigest[HEX_DIGEST256_LEN+1];
422  crypto_digest256((char *)sha3_digest,
423  (const char *)body, bodylen, DIGEST_SHA3_256);
424  base16_encode(hexdigest, sizeof(hexdigest),
425  (const char *)sha3_digest, sizeof(sha3_digest));
426 
427  config_line_prepend(labels, label, hexdigest);
428 }
429 
434 STATIC int
435 cdm_entry_get_sha3_value(uint8_t *digest_out,
437  const char *label)
438 {
439  if (ent == NULL)
440  return -1;
441 
442  const char *hex = consensus_cache_entry_get_value(ent, label);
443  if (hex == NULL)
444  return -1;
445 
446  int n = base16_decode((char*)digest_out, DIGEST256_LEN, hex, strlen(hex));
447  if (n != DIGEST256_LEN)
448  return -2;
449  else
450  return 0;
451 }
452 
459 cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after)
460 {
461  char formatted_time[ISO_TIME_LEN+1];
462  format_iso_time_nospace(formatted_time, valid_after);
463  const char *flavname = networkstatus_get_flavor_name(flavor);
464 
465  /* We'll filter by valid-after time first, since that should
466  * match the fewest documents. */
467  /* We could add an extra hashtable here, but since we only do this scan
468  * when adding a new consensus, it probably doesn't matter much. */
469  smartlist_t *matches = smartlist_new();
470  consensus_cache_find_all(matches, cdm_cache_get(),
471  LABEL_VALID_AFTER, formatted_time);
472  consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
473  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
474 
475  consensus_cache_entry_t *result = NULL;
476  if (smartlist_len(matches)) {
477  result = smartlist_get(matches, 0);
478  }
479  smartlist_free(matches);
480 
481  return result;
482 }
483 
487 static int32_t
488 get_max_age_to_cache(void)
489 {
490  const int32_t DEFAULT_MAX_AGE_TO_CACHE = 8192;
491  const int32_t MIN_MAX_AGE_TO_CACHE = 0;
492  const int32_t MAX_MAX_AGE_TO_CACHE = 8192;
493  const char MAX_AGE_TO_CACHE_NAME[] = "max-consensus-age-to-cache-for-diff";
494 
495  const or_options_t *options = get_options();
496 
497  if (options->MaxConsensusAgeForDiffs) {
498  const int v = options->MaxConsensusAgeForDiffs;
499  if (v >= MAX_MAX_AGE_TO_CACHE * 3600)
500  return MAX_MAX_AGE_TO_CACHE;
501  else
502  return v;
503  }
504 
505  /* The parameter is in hours, so we multiply */
506  return 3600 * networkstatus_get_param(NULL,
507  MAX_AGE_TO_CACHE_NAME,
508  DEFAULT_MAX_AGE_TO_CACHE,
509  MIN_MAX_AGE_TO_CACHE,
510  MAX_MAX_AGE_TO_CACHE);
511 }
512 
513 #ifdef TOR_UNIT_TESTS
514 
516 int
517 consdiffmgr_add_consensus_nulterm(const char *consensus,
518  const networkstatus_t *as_parsed)
519 {
520  size_t len = strlen(consensus);
521  /* make a non-nul-terminated copy so that we can have a better chance
522  * of catching errors. */
523  char *ctmp = tor_memdup(consensus, len);
524  int r = consdiffmgr_add_consensus(ctmp, len, as_parsed);
525  tor_free(ctmp);
526  return r;
527 }
528 #endif /* defined(TOR_UNIT_TESTS) */
529 
538 int
539 consdiffmgr_add_consensus(const char *consensus,
540  size_t consensus_len,
541  const networkstatus_t *as_parsed)
542 {
543  if (BUG(consensus == NULL) || BUG(as_parsed == NULL))
544  return -1; // LCOV_EXCL_LINE
545  if (BUG(as_parsed->type != NS_TYPE_CONSENSUS))
546  return -1; // LCOV_EXCL_LINE
547 
548  const consensus_flavor_t flavor = as_parsed->flavor;
549  const time_t valid_after = as_parsed->valid_after;
550 
551  if (valid_after < approx_time() - get_max_age_to_cache()) {
552  log_info(LD_DIRSERV, "We don't care about this consensus document; it's "
553  "too old.");
554  return -1;
555  }
556 
557  /* Do we already have this one? */
558  consensus_cache_entry_t *entry =
559  cdm_cache_lookup_consensus(flavor, valid_after);
560  if (entry) {
561  log_info(LD_DIRSERV, "We already have a copy of that consensus");
562  return -1;
563  }
564 
565  /* We don't have it. Add it to the cache. */
566  return consensus_queue_compression_work(consensus, consensus_len, as_parsed);
567 }
568 
573 static int
574 compare_by_valid_after_(const void **a, const void **b)
575 {
576  const consensus_cache_entry_t *e1 = *a;
577  const consensus_cache_entry_t *e2 = *b;
578  /* We're in luck here: sorting UTC iso-encoded values lexically will work
579  * fine (until 9999). */
580  return strcmp_opt(consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER),
581  consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER));
582 }
583 
589 sort_and_find_most_recent(smartlist_t *lst)
590 {
591  smartlist_sort(lst, compare_by_valid_after_);
592  if (smartlist_len(lst)) {
593  return smartlist_get(lst, smartlist_len(lst) - 1);
594  } else {
595  return NULL;
596  }
597 }
598 
601 static int
602 consensus_compression_method_pos(compress_method_t method)
603 {
604  unsigned i;
605  for (i = 0; i < n_consensus_compression_methods(); ++i) {
606  if (compress_consensus_with[i] == method) {
607  return i;
608  }
609  }
610  return -1;
611 }
612 
618 consdiff_status_t
619 consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out,
620  consensus_flavor_t flavor,
621  compress_method_t method)
622 {
623  tor_assert(entry_out);
624  tor_assert((int)flavor < N_CONSENSUS_FLAVORS);
625 
626  int pos = consensus_compression_method_pos(method);
627  if (pos < 0) {
628  // We don't compress consensuses with this method.
629  return CONSDIFF_NOT_FOUND;
630  }
631  consensus_cache_entry_handle_t *handle = latest_consensus[flavor][pos];
632  if (!handle)
633  return CONSDIFF_NOT_FOUND;
634  *entry_out = consensus_cache_entry_handle_get(handle);
635  if (*entry_out)
636  return CONSDIFF_AVAILABLE;
637  else
638  return CONSDIFF_NOT_FOUND;
639 }
640 
649 consdiff_status_t
650 consdiffmgr_find_diff_from(consensus_cache_entry_t **entry_out,
651  consensus_flavor_t flavor,
652  int digest_type,
653  const uint8_t *digest,
654  size_t digestlen,
655  compress_method_t method)
656 {
657  if (BUG(digest_type != DIGEST_SHA3_256) ||
658  BUG(digestlen != DIGEST256_LEN)) {
659  return CONSDIFF_NOT_FOUND; // LCOV_EXCL_LINE
660  }
661 
662  // Try to look up the entry in the hashtable.
663  cdm_diff_t search, *ent;
664  memset(&search, 0, sizeof(search));
665  search.flavor = flavor;
666  search.compress_method = method;
667  memcpy(search.from_sha3, digest, DIGEST256_LEN);
668  ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
669 
670  if (ent == NULL ||
671  ent->cdm_diff_status == CDM_DIFF_ERROR) {
672  return CONSDIFF_NOT_FOUND;
673  } else if (ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS) {
674  return CONSDIFF_IN_PROGRESS;
675  } else if (BUG(ent->cdm_diff_status != CDM_DIFF_PRESENT)) {
676  return CONSDIFF_IN_PROGRESS;
677  }
678 
679  if (BUG(ent->entry == NULL)) {
680  return CONSDIFF_NOT_FOUND;
681  }
682  *entry_out = consensus_cache_entry_handle_get(ent->entry);
683  return (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
684 
685 #if 0
686  // XXXX Remove this. I'm keeping it around for now in case we need to
687  // XXXX debug issues in the hashtable.
688  char hex[HEX_DIGEST256_LEN+1];
689  base16_encode(hex, sizeof(hex), (const char *)digest, digestlen);
690  const char *flavname = networkstatus_get_flavor_name(flavor);
691 
692  smartlist_t *matches = smartlist_new();
693  consensus_cache_find_all(matches, cdm_cache_get(),
694  LABEL_FROM_SHA3_DIGEST, hex);
695  consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
696  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
697 
698  *entry_out = sort_and_find_most_recent(matches);
699  consdiff_status_t result =
700  (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
701  smartlist_free(matches);
702 
703  return result;
704 #endif /* 0 */
705 }
706 
711 int
712 consdiffmgr_cleanup(void)
713 {
714  smartlist_t *objects = smartlist_new();
715  smartlist_t *consensuses = smartlist_new();
716  smartlist_t *diffs = smartlist_new();
717  int n_to_delete = 0;
718 
719  log_debug(LD_DIRSERV, "Looking for consdiffmgr entries to remove");
720 
721  // 1. Delete any consensus or diff or anything whose valid_after is too old.
722  const time_t valid_after_cutoff = approx_time() - get_max_age_to_cache();
723 
724  consensus_cache_find_all(objects, cdm_cache_get(),
725  NULL, NULL);
727  const char *lv_valid_after =
728  consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
729  if (! lv_valid_after) {
730  log_debug(LD_DIRSERV, "Ignoring entry because it had no %s label",
731  LABEL_VALID_AFTER);
732  continue;
733  }
734  time_t valid_after = 0;
735  if (parse_iso_time_nospace(lv_valid_after, &valid_after) < 0) {
736  log_debug(LD_DIRSERV, "Ignoring entry because its %s value (%s) was "
737  "unparseable", LABEL_VALID_AFTER, escaped(lv_valid_after));
738  continue;
739  }
740  if (valid_after < valid_after_cutoff) {
741  log_debug(LD_DIRSERV, "Deleting entry because its %s value (%s) was "
742  "too old", LABEL_VALID_AFTER, lv_valid_after);
743  consensus_cache_entry_mark_for_removal(ent);
744  ++n_to_delete;
745  }
746  } SMARTLIST_FOREACH_END(ent);
747 
748  // 2. Delete all diffs that lead to a consensus whose valid-after is not the
749  // latest.
750  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
751  const char *flavname = networkstatus_get_flavor_name(flav);
752  /* Determine the most recent consensus of this flavor */
753  consensus_cache_find_all(consensuses, cdm_cache_get(),
754  LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
755  consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
756  consensus_cache_entry_t *most_recent =
757  sort_and_find_most_recent(consensuses);
758  if (most_recent == NULL)
759  continue;
760  const char *most_recent_sha3 =
761  consensus_cache_entry_get_value(most_recent,
762  LABEL_SHA3_DIGEST_UNCOMPRESSED);
763  if (BUG(most_recent_sha3 == NULL))
764  continue; // LCOV_EXCL_LINE
765 
766  /* consider all such-flavored diffs, and look to see if they match. */
767  consensus_cache_find_all(diffs, cdm_cache_get(),
768  LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
769  consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
771  const char *this_diff_target_sha3 =
772  consensus_cache_entry_get_value(diff, LABEL_TARGET_SHA3_DIGEST);
773  if (!this_diff_target_sha3)
774  continue;
775  if (strcmp(this_diff_target_sha3, most_recent_sha3)) {
776  consensus_cache_entry_mark_for_removal(diff);
777  ++n_to_delete;
778  }
779  } SMARTLIST_FOREACH_END(diff);
780  smartlist_clear(consensuses);
781  smartlist_clear(diffs);
782  }
783 
784  // 3. Delete all consensuses except the most recent that are compressed with
785  // an un-preferred method.
786  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
787  const char *flavname = networkstatus_get_flavor_name(flav);
788  /* Determine the most recent consensus of this flavor */
789  consensus_cache_find_all(consensuses, cdm_cache_get(),
790  LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
791  consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
792  consensus_cache_entry_t *most_recent =
793  sort_and_find_most_recent(consensuses);
794  if (most_recent == NULL)
795  continue;
796  const char *most_recent_sha3_uncompressed =
797  consensus_cache_entry_get_value(most_recent,
798  LABEL_SHA3_DIGEST_UNCOMPRESSED);
799  const char *retain_methodname = compression_method_get_name(
800  RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
801 
802  if (BUG(most_recent_sha3_uncompressed == NULL))
803  continue;
804  SMARTLIST_FOREACH_BEGIN(consensuses, consensus_cache_entry_t *, ent) {
805  const char *lv_sha3_uncompressed =
806  consensus_cache_entry_get_value(ent, LABEL_SHA3_DIGEST_UNCOMPRESSED);
807  if (BUG(! lv_sha3_uncompressed))
808  continue;
809  if (!strcmp(lv_sha3_uncompressed, most_recent_sha3_uncompressed))
810  continue; // This _is_ the most recent.
811  const char *lv_methodname =
812  consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
813  if (! lv_methodname || strcmp(lv_methodname, retain_methodname)) {
814  consensus_cache_entry_mark_for_removal(ent);
815  ++n_to_delete;
816  }
817  } SMARTLIST_FOREACH_END(ent);
818  }
819 
820  smartlist_free(objects);
821  smartlist_free(consensuses);
822  smartlist_free(diffs);
823 
824  // Actually remove files, if they're not used.
825  consensus_cache_delete_pending(cdm_cache_get(), 0);
826  return n_to_delete;
827 }
828 
833 void
834 consdiffmgr_configure(const consdiff_cfg_t *cfg)
835 {
836  if (cfg)
837  memcpy(&consdiff_cfg, cfg, sizeof(consdiff_cfg));
838 
839  (void) cdm_cache_get();
840 }
841 
846 int
847 consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg)
848 {
849  return consensus_cache_register_with_sandbox(cdm_cache_get(), cfg);
850 }
851 
857 int
858 consdiffmgr_validate(void)
859 {
860  /* Right now, we only check for entries that have bad sha3 values */
861  int problems = 0;
862 
863  smartlist_t *objects = smartlist_new();
864  consensus_cache_find_all(objects, cdm_cache_get(),
865  NULL, NULL);
867  uint8_t sha3_expected[DIGEST256_LEN];
868  uint8_t sha3_received[DIGEST256_LEN];
869  int r = cdm_entry_get_sha3_value(sha3_expected, obj, LABEL_SHA3_DIGEST);
870  if (r == -1) {
871  /* digest isn't there; that's allowed */
872  continue;
873  } else if (r == -2) {
874  /* digest is malformed; that's not allowed */
875  problems = 1;
876  consensus_cache_entry_mark_for_removal(obj);
877  continue;
878  }
879  const uint8_t *body;
880  size_t bodylen;
881  consensus_cache_entry_incref(obj);
882  r = consensus_cache_entry_get_body(obj, &body, &bodylen);
883  if (r == 0) {
884  crypto_digest256((char *)sha3_received, (const char *)body, bodylen,
885  DIGEST_SHA3_256);
886  }
887  consensus_cache_entry_decref(obj);
888  if (r < 0)
889  continue;
890 
891  // Deconfuse coverity about the possibility of sha3_received being
892  // uninitialized
893  tor_assert(r <= 0);
894 
895  if (fast_memneq(sha3_received, sha3_expected, DIGEST256_LEN)) {
896  problems = 1;
897  consensus_cache_entry_mark_for_removal(obj);
898  continue;
899  }
900 
901  } SMARTLIST_FOREACH_END(obj);
902  smartlist_free(objects);
903  return problems;
904 }
905 
909 static void
910 consdiffmgr_rescan_flavor_(consensus_flavor_t flavor)
911 {
912  smartlist_t *matches = NULL;
913  smartlist_t *diffs = NULL;
914  smartlist_t *compute_diffs_from = NULL;
915  strmap_t *have_diff_from = NULL;
916 
917  // look for the most recent consensus, and for all previous in-range
918  // consensuses. Do they all have diffs to it?
919  const char *flavname = networkstatus_get_flavor_name(flavor);
920 
921  // 1. find the most recent consensus, and the ones that we might want
922  // to diff to it.
923  const char *methodname = compression_method_get_name(
924  RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
925 
926  matches = smartlist_new();
927  consensus_cache_find_all(matches, cdm_cache_get(),
928  LABEL_FLAVOR, flavname);
929  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
930  consensus_cache_filter_list(matches, LABEL_COMPRESSION_TYPE, methodname);
931  consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
932  if (!most_recent) {
933  log_info(LD_DIRSERV, "No 'most recent' %s consensus found; "
934  "not making diffs", flavname);
935  goto done;
936  }
937  tor_assert(smartlist_len(matches));
938  smartlist_del(matches, smartlist_len(matches) - 1);
939 
940  const char *most_recent_valid_after =
941  consensus_cache_entry_get_value(most_recent, LABEL_VALID_AFTER);
942  if (BUG(most_recent_valid_after == NULL))
943  goto done; //LCOV_EXCL_LINE
944  uint8_t most_recent_sha3[DIGEST256_LEN];
945  if (BUG(cdm_entry_get_sha3_value(most_recent_sha3, most_recent,
946  LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
947  goto done; //LCOV_EXCL_LINE
948 
949  // 2. Find all the relevant diffs _to_ this consensus. These are ones
950  // that we don't need to compute.
951  diffs = smartlist_new();
952  consensus_cache_find_all(diffs, cdm_cache_get(),
953  LABEL_VALID_AFTER, most_recent_valid_after);
954  consensus_cache_filter_list(diffs, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
955  consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
956  have_diff_from = strmap_new();
958  const char *va = consensus_cache_entry_get_value(diff,
959  LABEL_FROM_VALID_AFTER);
960  if (BUG(va == NULL))
961  continue; // LCOV_EXCL_LINE
962  strmap_set(have_diff_from, va, diff);
963  } SMARTLIST_FOREACH_END(diff);
964 
965  // 3. See which consensuses in 'matches' don't have diffs yet.
966  smartlist_reverse(matches); // from newest to oldest.
967  compute_diffs_from = smartlist_new();
969  const char *va = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
970  if (BUG(va == NULL))
971  continue; // LCOV_EXCL_LINE
972  if (strmap_get(have_diff_from, va) != NULL)
973  continue; /* we already have this one. */
974  smartlist_add(compute_diffs_from, ent);
975  /* Since we are not going to serve this as the most recent consensus
976  * any more, we should stop keeping it mmap'd when it's not in use.
977  */
978  consensus_cache_entry_mark_for_aggressive_release(ent);
979  } SMARTLIST_FOREACH_END(ent);
980 
981  log_info(LD_DIRSERV,
982  "The most recent %s consensus is valid-after %s. We have diffs to "
983  "this consensus for %d/%d older %s consensuses. Generating diffs "
984  "for the other %d.",
985  flavname,
986  most_recent_valid_after,
987  smartlist_len(matches) - smartlist_len(compute_diffs_from),
988  smartlist_len(matches),
989  flavname,
990  smartlist_len(compute_diffs_from));
991 
992  // 4. Update the hashtable; remove entries in this flavor to other
993  // target consensuses.
994  cdm_diff_ht_purge(flavor, most_recent_sha3);
995 
996  // 5. Actually launch the requests.
997  SMARTLIST_FOREACH_BEGIN(compute_diffs_from, consensus_cache_entry_t *, c) {
998  if (BUG(c == most_recent))
999  continue; // LCOV_EXCL_LINE
1000 
1001  uint8_t this_sha3[DIGEST256_LEN];
1002  if (cdm_entry_get_sha3_value(this_sha3, c,
1003  LABEL_SHA3_DIGEST_AS_SIGNED)<0) {
1004  // Not actually a bug, since we might be running with a directory
1005  // with stale files from before the #22143 fixes.
1006  continue;
1007  }
1008  if (cdm_diff_ht_check_and_note_pending(flavor,
1009  this_sha3, most_recent_sha3)) {
1010  // This is already pending, or we encountered an error.
1011  continue;
1012  }
1013  consensus_diff_queue_diff_work(c, most_recent);
1014  } SMARTLIST_FOREACH_END(c);
1015 
1016  done:
1017  smartlist_free(matches);
1018  smartlist_free(diffs);
1019  smartlist_free(compute_diffs_from);
1020  strmap_free(have_diff_from, NULL);
1021 }
1022 
1027 static void
1028 consdiffmgr_consensus_load(void)
1029 {
1030  smartlist_t *matches = smartlist_new();
1031  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
1032  const char *flavname = networkstatus_get_flavor_name(flav);
1033  smartlist_clear(matches);
1034  consensus_cache_find_all(matches, cdm_cache_get(),
1035  LABEL_FLAVOR, flavname);
1036  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
1037  consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
1038  if (! most_recent)
1039  continue; // no consensuses.
1040  const char *most_recent_sha3 =
1041  consensus_cache_entry_get_value(most_recent,
1042  LABEL_SHA3_DIGEST_UNCOMPRESSED);
1043  if (BUG(most_recent_sha3 == NULL))
1044  continue; // LCOV_EXCL_LINE
1045  consensus_cache_filter_list(matches, LABEL_SHA3_DIGEST_UNCOMPRESSED,
1046  most_recent_sha3);
1047 
1048  // Everything that remains matches the most recent consensus of this
1049  // flavor.
1051  const char *lv_compression =
1052  consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
1053  compress_method_t method =
1054  compression_method_get_by_name(lv_compression);
1055  int pos = consensus_compression_method_pos(method);
1056  if (pos < 0)
1057  continue;
1058  consensus_cache_entry_handle_free(latest_consensus[flav][pos]);
1059  latest_consensus[flav][pos] = consensus_cache_entry_handle_new(ent);
1060  } SMARTLIST_FOREACH_END(ent);
1061  }
1062  smartlist_free(matches);
1063 }
1064 
1068 static void
1069 consdiffmgr_diffs_load(void)
1070 {
1071  smartlist_t *diffs = smartlist_new();
1072  consensus_cache_find_all(diffs, cdm_cache_get(),
1073  LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
1075  const char *lv_flavor =
1076  consensus_cache_entry_get_value(diff, LABEL_FLAVOR);
1077  if (!lv_flavor)
1078  continue;
1079  int flavor = networkstatus_parse_flavor_name(lv_flavor);
1080  if (flavor < 0)
1081  continue;
1082  const char *lv_compression =
1083  consensus_cache_entry_get_value(diff, LABEL_COMPRESSION_TYPE);
1084  compress_method_t method = NO_METHOD;
1085  if (lv_compression) {
1086  method = compression_method_get_by_name(lv_compression);
1087  if (method == UNKNOWN_METHOD) {
1088  continue;
1089  }
1090  }
1091 
1092  uint8_t from_sha3[DIGEST256_LEN];
1093  uint8_t to_sha3[DIGEST256_LEN];
1094  if (cdm_entry_get_sha3_value(from_sha3, diff, LABEL_FROM_SHA3_DIGEST)<0)
1095  continue;
1096  if (cdm_entry_get_sha3_value(to_sha3, diff, LABEL_TARGET_SHA3_DIGEST)<0)
1097  continue;
1098 
1099  cdm_diff_ht_set_status(flavor, from_sha3, to_sha3,
1100  method,
1101  CDM_DIFF_PRESENT,
1102  consensus_cache_entry_handle_new(diff));
1103  } SMARTLIST_FOREACH_END(diff);
1104  smartlist_free(diffs);
1105 }
1106 
1110 void
1111 consdiffmgr_rescan(void)
1112 {
1113  if (cdm_cache_dirty == 0)
1114  return;
1115 
1116  // Clean up here to make room for new diffs, and to ensure that older
1117  // consensuses do not have any entries.
1118  consdiffmgr_cleanup();
1119 
1120  if (cdm_cache_loaded == 0) {
1121  consdiffmgr_diffs_load();
1122  consdiffmgr_consensus_load();
1123  cdm_cache_loaded = 1;
1124  }
1125 
1126  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
1127  consdiffmgr_rescan_flavor_((consensus_flavor_t) flav);
1128  }
1129 
1130  cdm_cache_dirty = 0;
1131 }
1132 
1134 static void
1135 consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg)
1136 {
1137  (void)ev;
1138  (void)arg;
1139  consdiffmgr_rescan();
1140 }
1141 
1143 static void
1144 mark_cdm_cache_dirty(void)
1145 {
1146  cdm_cache_dirty = 1;
1147  tor_assert(consdiffmgr_rescan_ev);
1148  mainloop_event_activate(consdiffmgr_rescan_ev);
1149 }
1150 
1157 static int
1158 compare_by_staleness_(const void **a, const void **b)
1159 {
1160  const consensus_cache_entry_t *e1 = *a;
1161  const consensus_cache_entry_t *e2 = *b;
1162  const char *va1, *fva1, *va2, *fva2;
1163  va1 = consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER);
1164  va2 = consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER);
1165  fva1 = consensus_cache_entry_get_value(e1, LABEL_FROM_VALID_AFTER);
1166  fva2 = consensus_cache_entry_get_value(e2, LABEL_FROM_VALID_AFTER);
1167 
1168  if (fva1)
1169  va1 = fva1;
1170  if (fva2)
1171  va2 = fva2;
1172 
1173  /* See note about iso-encoded values in compare_by_valid_after_. Also note
1174  * that missing dates will get placed first. */
1175  return strcmp_opt(va1, va2);
1176 }
1177 
1184 static int
1185 consdiffmgr_ensure_space_for_files(int n)
1186 {
1187  consensus_cache_t *cache = cdm_cache_get();
1188  if (consensus_cache_get_n_filenames_available(cache) >= n) {
1189  // there are already enough unused filenames.
1190  return 0;
1191  }
1192  // Try a cheap deletion of stuff that's waiting to get deleted.
1193  consensus_cache_delete_pending(cache, 0);
1194  if (consensus_cache_get_n_filenames_available(cache) >= n) {
1195  // okay, _that_ made enough filenames available.
1196  return 0;
1197  }
1198  // Let's get more assertive: clean out unused stuff, and force-remove
1199  // the files that we can.
1200  consdiffmgr_cleanup();
1201  consensus_cache_delete_pending(cache, 1);
1202  const int n_to_remove = n - consensus_cache_get_n_filenames_available(cache);
1203  if (n_to_remove <= 0) {
1204  // okay, finally!
1205  return 0;
1206  }
1207 
1208  // At this point, we're going to have to throw out objects that will be
1209  // missed. Too bad!
1210  smartlist_t *objects = smartlist_new();
1211  consensus_cache_find_all(objects, cache, NULL, NULL);
1212  smartlist_sort(objects, compare_by_staleness_);
1213  int n_marked = 0;
1215  consensus_cache_entry_mark_for_removal(ent);
1216  if (++n_marked >= n_to_remove)
1217  break;
1218  } SMARTLIST_FOREACH_END(ent);
1219  smartlist_free(objects);
1220 
1221  consensus_cache_delete_pending(cache, 1);
1222 
1223  if (consensus_cache_may_overallocate(cache)) {
1224  /* If we're allowed to throw extra files into the cache, let's do so
1225  * rather getting upset.
1226  */
1227  return 0;
1228  }
1229 
1230  if (BUG(n_marked < n_to_remove))
1231  return -1;
1232  else
1233  return 0;
1234 }
1235 
1239 static void
1240 consdiffmgr_set_cache_flags(void)
1241 {
1242  /* Right now, we just mark the consensus objects for aggressive release,
1243  * so that they get mmapped for as little time as possible. */
1244  smartlist_t *objects = smartlist_new();
1245  consensus_cache_find_all(objects, cdm_cache_get(), LABEL_DOCTYPE,
1246  DOCTYPE_CONSENSUS);
1248  consensus_cache_entry_mark_for_aggressive_release(ent);
1249  } SMARTLIST_FOREACH_END(ent);
1250  smartlist_free(objects);
1251 }
1252 
1256 void
1257 consdiffmgr_free_all(void)
1258 {
1259  cdm_diff_t **diff, **next;
1260  for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
1261  cdm_diff_t *this = *diff;
1262  next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
1263  cdm_diff_free(this);
1264  }
1265  int i;
1266  unsigned j;
1267  for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
1268  for (j = 0; j < n_consensus_compression_methods(); ++j) {
1269  consensus_cache_entry_handle_free(latest_consensus[i][j]);
1270  }
1271  }
1272  memset(latest_consensus, 0, sizeof(latest_consensus));
1273  consensus_cache_free(cons_diff_cache);
1274  cons_diff_cache = NULL;
1275  mainloop_event_free(consdiffmgr_rescan_ev);
1276 }
1277 
1278 /* =====
1279  Thread workers
1280  =====*/
1281 
1282 typedef struct compressed_result_t {
1283  config_line_t *labels;
1287  uint8_t *body;
1291  size_t bodylen;
1293 
1304 static int
1305 compress_multiple(compressed_result_t *results_out, int n_methods,
1306  const compress_method_t *methods,
1307  const uint8_t *input, size_t len,
1308  const config_line_t *labels_in)
1309 {
1310  int rv = 0;
1311  int i;
1312  for (i = 0; i < n_methods; ++i) {
1313  compress_method_t method = methods[i];
1314  const char *methodname = compression_method_get_name(method);
1315  char *result;
1316  size_t sz;
1317  if (0 == tor_compress(&result, &sz, (const char*)input, len, method)) {
1318  results_out[i].body = (uint8_t*)result;
1319  results_out[i].bodylen = sz;
1320  results_out[i].labels = config_lines_dup(labels_in);
1321  cdm_labels_prepend_sha3(&results_out[i].labels, LABEL_SHA3_DIGEST,
1322  results_out[i].body,
1323  results_out[i].bodylen);
1324  config_line_prepend(&results_out[i].labels,
1325  LABEL_COMPRESSION_TYPE,
1326  methodname);
1327  } else {
1328  rv = -1;
1329  }
1330  }
1331  return rv;
1332 }
1333 
1343 static cdm_diff_status_t
1344 store_multiple(consensus_cache_entry_handle_t **handles_out,
1345  int n,
1346  const compress_method_t *methods,
1347  const compressed_result_t *results,
1348  const char *description)
1349 {
1350  cdm_diff_status_t status = CDM_DIFF_ERROR;
1351  consdiffmgr_ensure_space_for_files(n);
1352 
1353  int i;
1354  for (i = 0; i < n; ++i) {
1355  compress_method_t method = methods[i];
1356  uint8_t *body_out = results[i].body;
1357  size_t bodylen_out = results[i].bodylen;
1358  config_line_t *labels = results[i].labels;
1359  const char *methodname = compression_method_get_name(method);
1360  if (body_out && bodylen_out && labels) {
1361  /* Success! Store the results */
1362  log_info(LD_DIRSERV, "Adding %s, compressed with %s",
1363  description, methodname);
1364 
1366  consensus_cache_add(cdm_cache_get(),
1367  labels,
1368  body_out,
1369  bodylen_out);
1370  if (ent == NULL) {
1371  static ratelim_t cant_store_ratelim = RATELIM_INIT(5*60);
1372  log_fn_ratelim(&cant_store_ratelim, LOG_WARN, LD_FS,
1373  "Unable to store object %s compressed with %s.",
1374  description, methodname);
1375  continue;
1376  }
1377 
1378  status = CDM_DIFF_PRESENT;
1379  handles_out[i] = consensus_cache_entry_handle_new(ent);
1380  consensus_cache_entry_decref(ent);
1381  }
1382  }
1383  return status;
1384 }
1385 
1403 
1405  compressed_result_t out[ARRAY_LENGTH(compress_diffs_with)];
1407 
1414 STATIC int
1415 uncompress_or_set_ptr(const char **out, size_t *outlen,
1416  char **owned_out,
1418 {
1419  const uint8_t *body;
1420  size_t bodylen;
1421 
1422  *owned_out = NULL;
1423 
1424  if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0)
1425  return -1;
1426 
1427  const char *lv_compression =
1428  consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
1429  compress_method_t method = NO_METHOD;
1430 
1431  if (lv_compression)
1432  method = compression_method_get_by_name(lv_compression);
1433 
1434  int rv;
1435  if (method == NO_METHOD) {
1436  *out = (const char *)body;
1437  *outlen = bodylen;
1438  rv = 0;
1439  } else {
1440  rv = tor_uncompress(owned_out, outlen, (const char *)body, bodylen,
1441  method, 1, LOG_WARN);
1442  *out = *owned_out;
1443  }
1444  return rv;
1445 }
1446 
1451 static workqueue_reply_t
1452 consensus_diff_worker_threadfn(void *state_, void *work_)
1453 {
1454  (void)state_;
1455  consensus_diff_worker_job_t *job = work_;
1456  const uint8_t *diff_from, *diff_to;
1457  size_t len_from, len_to;
1458  int r;
1459  /* We need to have the body already mapped into RAM here.
1460  */
1461  r = consensus_cache_entry_get_body(job->diff_from, &diff_from, &len_from);
1462  if (BUG(r < 0))
1463  return WQ_RPL_REPLY; // LCOV_EXCL_LINE
1464  r = consensus_cache_entry_get_body(job->diff_to, &diff_to, &len_to);
1465  if (BUG(r < 0))
1466  return WQ_RPL_REPLY; // LCOV_EXCL_LINE
1467 
1468  const char *lv_to_valid_after =
1469  consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER);
1470  const char *lv_to_fresh_until =
1471  consensus_cache_entry_get_value(job->diff_to, LABEL_FRESH_UNTIL);
1472  const char *lv_to_valid_until =
1473  consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_UNTIL);
1474  const char *lv_to_signatories =
1475  consensus_cache_entry_get_value(job->diff_to, LABEL_SIGNATORIES);
1476  const char *lv_from_valid_after =
1477  consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER);
1478  const char *lv_from_digest =
1479  consensus_cache_entry_get_value(job->diff_from,
1480  LABEL_SHA3_DIGEST_AS_SIGNED);
1481  const char *lv_from_flavor =
1482  consensus_cache_entry_get_value(job->diff_from, LABEL_FLAVOR);
1483  const char *lv_to_flavor =
1484  consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
1485  const char *lv_to_digest =
1486  consensus_cache_entry_get_value(job->diff_to,
1487  LABEL_SHA3_DIGEST_UNCOMPRESSED);
1488 
1489  if (! lv_from_digest) {
1490  /* This isn't a bug right now, since it can happen if you're migrating
1491  * from an older version of master to a newer one. The older ones didn't
1492  * annotate their stored consensus objects with sha3-digest-as-signed.
1493  */
1494  return WQ_RPL_REPLY; // LCOV_EXCL_LINE
1495  }
1496 
1497  /* All these values are mandatory on the input */
1498  if (BUG(!lv_to_valid_after) ||
1499  BUG(!lv_from_valid_after) ||
1500  BUG(!lv_from_flavor) ||
1501  BUG(!lv_to_flavor)) {
1502  return WQ_RPL_REPLY; // LCOV_EXCL_LINE
1503  }
1504  /* The flavors need to match */
1505  if (BUG(strcmp(lv_from_flavor, lv_to_flavor))) {
1506  return WQ_RPL_REPLY; // LCOV_EXCL_LINE
1507  }
1508 
1509  char *consensus_diff;
1510  {
1511  const char *diff_from_nt = NULL, *diff_to_nt = NULL;
1512  char *owned1 = NULL, *owned2 = NULL;
1513  size_t diff_from_nt_len, diff_to_nt_len;
1514 
1515  if (uncompress_or_set_ptr(&diff_from_nt, &diff_from_nt_len, &owned1,
1516  job->diff_from) < 0) {
1517  return WQ_RPL_REPLY;
1518  }
1519  if (uncompress_or_set_ptr(&diff_to_nt, &diff_to_nt_len, &owned2,
1520  job->diff_to) < 0) {
1521  tor_free(owned1);
1522  return WQ_RPL_REPLY;
1523  }
1524  tor_assert(diff_from_nt);
1525  tor_assert(diff_to_nt);
1526 
1527  // XXXX ugh; this is going to calculate the SHA3 of both its
1528  // XXXX inputs again, even though we already have that. Maybe it's time
1529  // XXXX to change the API here?
1530  consensus_diff = consensus_diff_generate(diff_from_nt,
1531  diff_from_nt_len,
1532  diff_to_nt,
1533  diff_to_nt_len);
1534  tor_free(owned1);
1535  tor_free(owned2);
1536  }
1537  if (!consensus_diff) {
1538  /* Couldn't generate consensus; we'll leave the reply blank. */
1539  return WQ_RPL_REPLY;
1540  }
1541 
1542  /* Compress the results and send the reply */
1543  tor_assert(compress_diffs_with[0] == NO_METHOD);
1544  size_t difflen = strlen(consensus_diff);
1545  job->out[0].body = (uint8_t *) consensus_diff;
1546  job->out[0].bodylen = difflen;
1547 
1548  config_line_t *common_labels = NULL;
1549  if (lv_to_valid_until)
1550  config_line_prepend(&common_labels, LABEL_VALID_UNTIL, lv_to_valid_until);
1551  if (lv_to_fresh_until)
1552  config_line_prepend(&common_labels, LABEL_FRESH_UNTIL, lv_to_fresh_until);
1553  if (lv_to_signatories)
1554  config_line_prepend(&common_labels, LABEL_SIGNATORIES, lv_to_signatories);
1555  cdm_labels_prepend_sha3(&common_labels,
1556  LABEL_SHA3_DIGEST_UNCOMPRESSED,
1557  job->out[0].body,
1558  job->out[0].bodylen);
1559  config_line_prepend(&common_labels, LABEL_FROM_VALID_AFTER,
1560  lv_from_valid_after);
1561  config_line_prepend(&common_labels, LABEL_VALID_AFTER,
1562  lv_to_valid_after);
1563  config_line_prepend(&common_labels, LABEL_FLAVOR, lv_from_flavor);
1564  config_line_prepend(&common_labels, LABEL_FROM_SHA3_DIGEST,
1565  lv_from_digest);
1566  config_line_prepend(&common_labels, LABEL_TARGET_SHA3_DIGEST,
1567  lv_to_digest);
1568  config_line_prepend(&common_labels, LABEL_DOCTYPE,
1569  DOCTYPE_CONSENSUS_DIFF);
1570 
1571  job->out[0].labels = config_lines_dup(common_labels);
1572  cdm_labels_prepend_sha3(&job->out[0].labels,
1573  LABEL_SHA3_DIGEST,
1574  job->out[0].body,
1575  job->out[0].bodylen);
1576 
1577  compress_multiple(job->out+1,
1578  n_diff_compression_methods()-1,
1579  compress_diffs_with+1,
1580  (const uint8_t*)consensus_diff, difflen, common_labels);
1581 
1582  config_free_lines(common_labels);
1583  return WQ_RPL_REPLY;
1584 }
1585 
1586 #define consensus_diff_worker_job_free(job) \
1587  FREE_AND_NULL(consensus_diff_worker_job_t, \
1588  consensus_diff_worker_job_free_, (job))
1589 
1593 static void
1594 consensus_diff_worker_job_free_(consensus_diff_worker_job_t *job)
1595 {
1596  if (!job)
1597  return;
1598  unsigned u;
1599  for (u = 0; u < n_diff_compression_methods(); ++u) {
1600  config_free_lines(job->out[u].labels);
1601  tor_free(job->out[u].body);
1602  }
1603  consensus_cache_entry_decref(job->diff_from);
1604  consensus_cache_entry_decref(job->diff_to);
1605  tor_free(job);
1606 }
1607 
1613 static void
1614 consensus_diff_worker_replyfn(void *work_)
1615 {
1617  tor_assert(work_);
1618 
1619  consensus_diff_worker_job_t *job = work_;
1620 
1621  const char *lv_from_digest =
1622  consensus_cache_entry_get_value(job->diff_from,
1623  LABEL_SHA3_DIGEST_AS_SIGNED);
1624  const char *lv_to_digest =
1625  consensus_cache_entry_get_value(job->diff_to,
1626  LABEL_SHA3_DIGEST_UNCOMPRESSED);
1627  const char *lv_flavor =
1628  consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
1629  if (BUG(lv_from_digest == NULL))
1630  lv_from_digest = "???"; // LCOV_EXCL_LINE
1631  if (BUG(lv_to_digest == NULL))
1632  lv_to_digest = "???"; // LCOV_EXCL_LINE
1633 
1634  uint8_t from_sha3[DIGEST256_LEN];
1635  uint8_t to_sha3[DIGEST256_LEN];
1636  int flav = -1;
1637  int cache = 1;
1638  if (BUG(cdm_entry_get_sha3_value(from_sha3, job->diff_from,
1639  LABEL_SHA3_DIGEST_AS_SIGNED) < 0))
1640  cache = 0;
1641  if (BUG(cdm_entry_get_sha3_value(to_sha3, job->diff_to,
1642  LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
1643  cache = 0;
1644  if (BUG(lv_flavor == NULL)) {
1645  cache = 0;
1646  } else if ((flav = networkstatus_parse_flavor_name(lv_flavor)) < 0) {
1647  cache = 0;
1648  }
1649 
1650  consensus_cache_entry_handle_t *handles[ARRAY_LENGTH(compress_diffs_with)];
1651  memset(handles, 0, sizeof(handles));
1652 
1653  char description[128];
1654  tor_snprintf(description, sizeof(description),
1655  "consensus diff from %s to %s",
1656  lv_from_digest, lv_to_digest);
1657 
1658  int status = store_multiple(handles,
1659  n_diff_compression_methods(),
1660  compress_diffs_with,
1661  job->out,
1662  description);
1663 
1664  if (status != CDM_DIFF_PRESENT) {
1665  /* Failure! Nothing to do but complain */
1666  log_warn(LD_DIRSERV,
1667  "Worker was unable to compute consensus diff "
1668  "from %s to %s", lv_from_digest, lv_to_digest);
1669  /* Cache this error so we don't try to compute this one again. */
1670  status = CDM_DIFF_ERROR;
1671  }
1672 
1673  unsigned u;
1674  for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
1675  compress_method_t method = compress_diffs_with[u];
1676  if (cache) {
1677  consensus_cache_entry_handle_t *h = handles[u];
1678  int this_status = status;
1679  if (h == NULL) {
1680  this_status = CDM_DIFF_ERROR;
1681  }
1682  tor_assert_nonfatal(h != NULL || this_status == CDM_DIFF_ERROR);
1683  cdm_diff_ht_set_status(flav, from_sha3, to_sha3, method, this_status, h);
1684  } else {
1685  consensus_cache_entry_handle_free(handles[u]);
1686  }
1687  }
1688 
1689  consensus_diff_worker_job_free(job);
1690 }
1691 
1696 static int
1697 consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
1698  consensus_cache_entry_t *diff_to)
1699 {
1701 
1702  consensus_cache_entry_incref(diff_from);
1703  consensus_cache_entry_incref(diff_to);
1704 
1705  consensus_diff_worker_job_t *job = tor_malloc_zero(sizeof(*job));
1706  job->diff_from = diff_from;
1707  job->diff_to = diff_to;
1708 
1709  /* Make sure body is mapped. */
1710  const uint8_t *body;
1711  size_t bodylen;
1712  int r1 = consensus_cache_entry_get_body(diff_from, &body, &bodylen);
1713  int r2 = consensus_cache_entry_get_body(diff_to, &body, &bodylen);
1714  if (r1 < 0 || r2 < 0)
1715  goto err;
1716 
1717  workqueue_entry_t *work;
1718  work = cpuworker_queue_work(WQ_PRI_LOW,
1719  consensus_diff_worker_threadfn,
1720  consensus_diff_worker_replyfn,
1721  job);
1722  if (!work)
1723  goto err;
1724 
1725  return 0;
1726  err:
1727  consensus_diff_worker_job_free(job); // includes decrefs.
1728  return -1;
1729 }
1730 
1735  char *consensus;
1736  size_t consensus_len;
1737  consensus_flavor_t flavor;
1738  config_line_t *labels_in;
1739  compressed_result_t out[ARRAY_LENGTH(compress_consensus_with)];
1741 
1742 #define consensus_compress_worker_job_free(job) \
1743  FREE_AND_NULL(consensus_compress_worker_job_t, \
1744  consensus_compress_worker_job_free_, (job))
1745 
1749 static void
1750 consensus_compress_worker_job_free_(consensus_compress_worker_job_t *job)
1751 {
1752  if (!job)
1753  return;
1754  tor_free(job->consensus);
1755  config_free_lines(job->labels_in);
1756  unsigned u;
1757  for (u = 0; u < n_consensus_compression_methods(); ++u) {
1758  config_free_lines(job->out[u].labels);
1759  tor_free(job->out[u].body);
1760  }
1761  tor_free(job);
1762 }
1767 static workqueue_reply_t
1768 consensus_compress_worker_threadfn(void *state_, void *work_)
1769 {
1770  (void)state_;
1771  consensus_compress_worker_job_t *job = work_;
1772  consensus_flavor_t flavor = job->flavor;
1773  const char *consensus = job->consensus;
1774  size_t bodylen = job->consensus_len;
1775 
1776  config_line_t *labels = config_lines_dup(job->labels_in);
1777  const char *flavname = networkstatus_get_flavor_name(flavor);
1778 
1779  cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED,
1780  (const uint8_t *)consensus, bodylen);
1781  {
1782  const char *start, *end;
1783  if (router_get_networkstatus_v3_signed_boundaries(consensus, bodylen,
1784  &start, &end) < 0) {
1785  start = consensus;
1786  end = consensus+bodylen;
1787  }
1788  cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED,
1789  (const uint8_t *)start,
1790  end - start);
1791  }
1792  config_line_prepend(&labels, LABEL_FLAVOR, flavname);
1793  config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
1794 
1795  compress_multiple(job->out,
1796  n_consensus_compression_methods(),
1797  compress_consensus_with,
1798  (const uint8_t*)consensus, bodylen, labels);
1799  config_free_lines(labels);
1800  return WQ_RPL_REPLY;
1801 }
1802 
1808 static void
1809 consensus_compress_worker_replyfn(void *work_)
1810 {
1811  consensus_compress_worker_job_t *job = work_;
1812 
1813  consensus_cache_entry_handle_t *handles[
1814  ARRAY_LENGTH(compress_consensus_with)];
1815  memset(handles, 0, sizeof(handles));
1816 
1817  store_multiple(handles,
1818  n_consensus_compression_methods(),
1819  compress_consensus_with,
1820  job->out,
1821  "consensus");
1822  mark_cdm_cache_dirty();
1823 
1824  unsigned u;
1825  consensus_flavor_t f = job->flavor;
1826  tor_assert((int)f < N_CONSENSUS_FLAVORS);
1827  for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
1828  if (handles[u] == NULL)
1829  continue;
1830  consensus_cache_entry_handle_free(latest_consensus[f][u]);
1831  latest_consensus[f][u] = handles[u];
1832  }
1833 
1834  consensus_compress_worker_job_free(job);
1835 }
1836 
1840 static int background_compression = 0;
1841 
1846 static int
1847 consensus_queue_compression_work(const char *consensus,
1848  size_t consensus_len,
1849  const networkstatus_t *as_parsed)
1850 {
1851  tor_assert(consensus);
1852  tor_assert(as_parsed);
1853 
1854  consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job));
1855  job->consensus = tor_memdup_nulterm(consensus, consensus_len);
1856  job->consensus_len = strlen(job->consensus);
1857  job->flavor = as_parsed->flavor;
1858 
1859  char va_str[ISO_TIME_LEN+1];
1860  char vu_str[ISO_TIME_LEN+1];
1861  char fu_str[ISO_TIME_LEN+1];
1862  format_iso_time_nospace(va_str, as_parsed->valid_after);
1863  format_iso_time_nospace(fu_str, as_parsed->fresh_until);
1864  format_iso_time_nospace(vu_str, as_parsed->valid_until);
1865  config_line_append(&job->labels_in, LABEL_VALID_AFTER, va_str);
1866  config_line_append(&job->labels_in, LABEL_FRESH_UNTIL, fu_str);
1867  config_line_append(&job->labels_in, LABEL_VALID_UNTIL, vu_str);
1868  if (as_parsed->voters) {
1869  smartlist_t *hexvoters = smartlist_new();
1870  SMARTLIST_FOREACH_BEGIN(as_parsed->voters,
1872  if (smartlist_len(vi->sigs) == 0)
1873  continue; // didn't sign.
1874  char d[HEX_DIGEST_LEN+1];
1875  base16_encode(d, sizeof(d), vi->identity_digest, DIGEST_LEN);
1876  smartlist_add_strdup(hexvoters, d);
1877  } SMARTLIST_FOREACH_END(vi);
1878  char *signers = smartlist_join_strings(hexvoters, ",", 0, NULL);
1879  config_line_prepend(&job->labels_in, LABEL_SIGNATORIES, signers);
1880  tor_free(signers);
1881  SMARTLIST_FOREACH(hexvoters, char *, cp, tor_free(cp));
1882  smartlist_free(hexvoters);
1883  }
1884 
1885  if (background_compression) {
1886  workqueue_entry_t *work;
1887  work = cpuworker_queue_work(WQ_PRI_LOW,
1888  consensus_compress_worker_threadfn,
1889  consensus_compress_worker_replyfn,
1890  job);
1891  if (!work) {
1892  consensus_compress_worker_job_free(job);
1893  return -1;
1894  }
1895 
1896  return 0;
1897  } else {
1898  consensus_compress_worker_threadfn(NULL, job);
1899  consensus_compress_worker_replyfn(job);
1900  return 0;
1901  }
1902 }
1903 
1907 void
1908 consdiffmgr_enable_background_compression(void)
1909 {
1910  // This isn't the default behavior because it would break unit tests.
1911  background_compression = 1;
1912 }
1913 
1917 int
1918 consensus_cache_entry_get_voter_id_digests(const consensus_cache_entry_t *ent,
1919  smartlist_t *out)
1920 {
1921  tor_assert(ent);
1922  tor_assert(out);
1923  const char *s;
1924  s = consensus_cache_entry_get_value(ent, LABEL_SIGNATORIES);
1925  if (s == NULL)
1926  return -1;
1927  smartlist_split_string(out, s, ",", SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE, 0);
1928  return 0;
1929 }
1930 
1933 int
1934 consensus_cache_entry_get_fresh_until(const consensus_cache_entry_t *ent,
1935  time_t *out)
1936 {
1937  tor_assert(ent);
1938  tor_assert(out);
1939  const char *s;
1940  s = consensus_cache_entry_get_value(ent, LABEL_FRESH_UNTIL);
1941  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
1942  return -1;
1943  else
1944  return 0;
1945 }
1946 
1949 int
1950 consensus_cache_entry_get_valid_until(const consensus_cache_entry_t *ent,
1951  time_t *out)
1952 {
1953  tor_assert(ent);
1954  tor_assert(out);
1955 
1956  const char *s;
1957  s = consensus_cache_entry_get_value(ent, LABEL_VALID_UNTIL);
1958  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
1959  return -1;
1960  else
1961  return 0;
1962 }
1963 
1966 int
1967 consensus_cache_entry_get_valid_after(const consensus_cache_entry_t *ent,
1968  time_t *out)
1969 {
1970  tor_assert(ent);
1971  tor_assert(out);
1972 
1973  const char *s;
1974  s = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
1975 
1976  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
1977  return -1;
1978  else
1979  return 0;
1980 }
networkstatus_type_t type
int MaxConsensusAgeForDiffs
Header for confline.c.
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
void smartlist_add_strdup(struct smartlist_t *sl, const char *string)
Headers for compress.c.
workqueue_reply_t
Definition: workqueue.h:24
HT_PROTOTYPE(HT_GENERATE2(strmap_impl, HT_GENERATE2(strmap_entry_t, HT_GENERATE2(node, HT_GENERATE2(strmap_entry_hash, HT_GENERATE2(strmap_entries_eq)
Definition: map.c:87
void smartlist_add(smartlist_t *sl, void *element)
Header file for config.c.
#define HEX_DIGEST256_LEN
Definition: crypto_digest.h:37
char * consensus_diff_generate(const char *cons1, size_t cons1len, const char *cons2, size_t cons2len)
Definition: consdiff.c:1339
Header file for cpuworker.c.
#define tor_free(p)
Definition: malloc.h:52
int parse_iso_time_nospace(const char *cp, time_t *t)
Definition: time_fmt.c:401
mainloop_event_t * mainloop_event_postloop_new(void(*cb)(mainloop_event_t *, void *), void *userdata)
Definition: conscache.c:27
void mainloop_event_activate(mainloop_event_t *event)
int tor_uncompress(char **out, size_t *out_len, const char *in, size_t in_len, compress_method_t method, int complete_only, int protocol_warn_level)
Definition: compress.c:268
int in_main_thread(void)
int networkstatus_parse_flavor_name(const char *flavname)
#define DIGEST256_LEN
Definition: digest_sizes.h:23
const char * compression_method_get_name(compress_method_t method)
Definition: compress.c:364
tor_assert(buffer)
int router_get_networkstatus_v3_signed_boundaries(const char *s, size_t len, const char **start_out, const char **end_out)
Definition: ns_parse.c:153
int tor_compress(char **out, size_t *out_len, const char *in, size_t in_len, compress_method_t method)
Definition: compress.c:242
consensus_cache_entry_t * diff_from
Definition: consdiffmgr.c:1396
#define DIGEST_LEN
Definition: digest_sizes.h:20
Master header file for Tor-specific functionality.
#define LD_DIRSERV
Definition: log.h:88
void smartlist_reverse(smartlist_t *sl)
Definition: smartlist.c:59
#define LOG_WARN
Definition: log.h:51
#define log_fn_ratelim(ratelim, severity, domain, args,...)
Definition: log.h:279
void format_iso_time_nospace(char *buf, time_t t)
Definition: time_fmt.c:313
void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:478
void tor_free_(void *mem)
Definition: malloc.c:227
void smartlist_del(smartlist_t *sl, int idx)
smartlist_t * voters
#define LD_FS
Definition: log.h:68
#define HEX_DIGEST_LEN
Definition: crypto_digest.h:35
consensus_flavor_t flavor
config_line_t * config_lines_dup(const config_line_t *inp)
Definition: confline.c:227
void config_line_prepend(config_line_t **lst, const char *key, const char *val)
Definition: confline.c:53
Header file for ns_parse.c.
int tor_snprintf(char *str, size_t size, const char *format,...)
Definition: printf.c:27
char * smartlist_join_strings(smartlist_t *sl, const char *join, int terminate, size_t *len_out)
Definition: smartlist.c:279
Definition: workqueue.c:95
#define SMARTLIST_FOREACH(sl, type, var, cmd)
const char * escaped(const char *s)
Definition: escape.c:126
consensus_flavor_t
Definition: or.h:867
#define ARRAY_LENGTH(x)
compressed_result_t out[ARRAY_LENGTH(compress_diffs_with)]
Definition: consdiffmgr.c:1405
void config_line_append(config_line_t **lst, const char *key, const char *val)
Definition: confline.c:32
time_t approx_time(void)
Definition: approx_time.c:32
consensus_cache_entry_t * diff_to
Definition: consdiffmgr.c:1402
int strcmp_opt(const char *s1, const char *s2)
Definition: util_string.c:188
#define N_CONSENSUS_FLAVORS
Definition: or.h:873
compress_method_t compression_method_get_by_name(const char *name)
Definition: compress.c:403
int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:506
compress_method_t
Definition: compress.h:21
int crypto_digest256(char *digest, const char *m, size_t len, digest_algorithm_t algorithm)
Header for compat_libevent.c.
Header for workqueue.c.
const char * networkstatus_get_flavor_name(consensus_flavor_t flav)
void smartlist_clear(smartlist_t *sl)
void smartlist_sort(smartlist_t *sl, int(*compare)(const void **a, const void **b))
Definition: smartlist.c:334
Header file for networkstatus.c.
int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep, int flags, int max)