14 #define CONSDIFFMGR_PRIVATE
38 #define LABEL_DOCTYPE "document-type"
41 #define LABEL_VALID_AFTER "consensus-valid-after"
44 #define LABEL_FRESH_UNTIL "consensus-fresh-until"
47 #define LABEL_VALID_UNTIL "consensus-valid-until"
50 #define LABEL_SIGNATORIES "consensus-signatories"
52 #define LABEL_SHA3_DIGEST "sha3-digest"
54 #define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed"
56 #define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed"
58 #define LABEL_FLAVOR "consensus-flavor"
60 #define LABEL_FROM_SHA3_DIGEST "from-sha3-digest"
62 #define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest"
64 #define LABEL_FROM_VALID_AFTER "from-valid-after"
66 #define LABEL_COMPRESSION_TYPE "compression"
69 #define DOCTYPE_CONSENSUS "consensus"
70 #define DOCTYPE_CONSENSUS_DIFF "consensus-diff"
92 CDM_DIFF_IN_PROGRESS=2,
145 #define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD
149 static consensus_cache_entry_handle_t *
174 consensus_cache_entry_handle_t *entry;
192 size_t consensus_len,
210 return (
unsigned) siphash24g(tmp,
sizeof(tmp));
217 diff1->flavor == diff2->flavor &&
218 diff1->compress_method == diff2->compress_method;
225 #define cdm_diff_free(diff) \
226 FREE_AND_NULL(cdm_diff_t, cdm_diff_free_, (diff))
234 consensus_cache_entry_handle_free(diff->entry);
242 const uint8_t *from_sha3,
243 const uint8_t *target_sha3,
251 ent->compress_method = method;
265 const uint8_t *from_sha3,
266 const uint8_t *target_sha3)
274 search.flavor = flav;
275 search.compress_method = method;
277 ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
279 tor_assert_nonfatal(ent->cdm_diff_status != CDM_DIFF_PRESENT);
283 ent =
cdm_diff_new(flav, from_sha3, target_sha3, method);
284 ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
285 HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
298 const uint8_t *from_sha3,
299 const uint8_t *to_sha3,
302 consensus_cache_entry_handle_t *handle)
304 if (handle == NULL) {
305 tor_assert_nonfatal(status != CDM_DIFF_PRESENT);
310 search.flavor = flav;
311 search.compress_method = method,
313 ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
316 ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
317 HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
325 tor_assert_nonfatal(ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS);
327 ent->cdm_diff_status = status;
328 consensus_cache_entry_handle_free(ent->entry);
342 const uint8_t *unless_target_sha3_matches)
345 for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
348 if ((*diff)->cdm_diff_status == CDM_DIFF_PRESENT &&
349 flav == (*diff)->flavor) {
351 if (BUG((*diff)->entry == NULL) ||
352 consensus_cache_entry_handle_get((*diff)->entry) == NULL) {
354 next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
359 if (unless_target_sha3_matches &&
360 fast_memneq(unless_target_sha3_matches, (*diff)->target_sha3,
363 next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
368 next = HT_NEXT(cdm_diff_ht, &cdm_diff_ht, diff);
378 unsigned n_entries = consdiff_cfg.cache_max_num * 2;
384 log_err(
LD_FS,
"Error: Couldn't open storage for consensus diffs.");
385 tor_assert_unreached();
423 (
const char *)body, bodylen, DIGEST_SHA3_256);
425 (
const char *)sha3_digest,
sizeof(sha3_digest));
461 char formatted_time[ISO_TIME_LEN+1];
471 LABEL_VALID_AFTER, formatted_time);
476 if (smartlist_len(matches)) {
477 result = smartlist_get(matches, 0);
479 smartlist_free(matches);
490 const int32_t DEFAULT_MAX_AGE_TO_CACHE = 8192;
491 const int32_t MIN_MAX_AGE_TO_CACHE = 0;
492 const int32_t MAX_MAX_AGE_TO_CACHE = 8192;
493 const char MAX_AGE_TO_CACHE_NAME[] =
"max-consensus-age-to-cache-for-diff";
499 if (v >= MAX_MAX_AGE_TO_CACHE * 3600)
500 return MAX_MAX_AGE_TO_CACHE;
507 MAX_AGE_TO_CACHE_NAME,
508 DEFAULT_MAX_AGE_TO_CACHE,
509 MIN_MAX_AGE_TO_CACHE,
510 MAX_MAX_AGE_TO_CACHE);
513 #ifdef TOR_UNIT_TESTS
517 consdiffmgr_add_consensus_nulterm(
const char *consensus,
520 size_t len = strlen(consensus);
523 char *ctmp = tor_memdup(consensus, len);
540 size_t consensus_len,
543 if (BUG(consensus == NULL) || BUG(as_parsed == NULL))
545 if (BUG(as_parsed->
type != NS_TYPE_CONSENSUS))
552 log_info(
LD_DIRSERV,
"We don't care about this consensus document; it's "
561 log_info(
LD_DIRSERV,
"We already have a copy of that consensus");
592 if (smartlist_len(lst)) {
593 return smartlist_get(lst, smartlist_len(lst) - 1);
629 return CONSDIFF_NOT_FOUND;
633 return CONSDIFF_NOT_FOUND;
634 *entry_out = consensus_cache_entry_handle_get(handle);
636 return CONSDIFF_AVAILABLE;
638 return CONSDIFF_NOT_FOUND;
653 const uint8_t *digest,
657 if (BUG(digest_type != DIGEST_SHA3_256) ||
659 return CONSDIFF_NOT_FOUND;
664 memset(&search, 0,
sizeof(search));
665 search.flavor = flavor;
666 search.compress_method = method;
668 ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
671 ent->cdm_diff_status == CDM_DIFF_ERROR) {
672 return CONSDIFF_NOT_FOUND;
673 }
else if (ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS) {
674 return CONSDIFF_IN_PROGRESS;
675 }
else if (BUG(ent->cdm_diff_status != CDM_DIFF_PRESENT)) {
676 return CONSDIFF_IN_PROGRESS;
679 if (BUG(ent->entry == NULL)) {
680 return CONSDIFF_NOT_FOUND;
682 *entry_out = consensus_cache_entry_handle_get(ent->entry);
683 return (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
689 base16_encode(hex,
sizeof(hex), (
const char *)digest, digestlen);
694 LABEL_FROM_SHA3_DIGEST, hex);
700 (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
701 smartlist_free(matches);
719 log_debug(
LD_DIRSERV,
"Looking for consdiffmgr entries to remove");
727 const char *lv_valid_after =
729 if (! lv_valid_after) {
730 log_debug(
LD_DIRSERV,
"Ignoring entry because it had no %s label",
734 time_t valid_after = 0;
736 log_debug(
LD_DIRSERV,
"Ignoring entry because its %s value (%s) was "
737 "unparseable", LABEL_VALID_AFTER,
escaped(lv_valid_after));
740 if (valid_after < valid_after_cutoff) {
741 log_debug(
LD_DIRSERV,
"Deleting entry because its %s value (%s) was "
742 "too old", LABEL_VALID_AFTER, lv_valid_after);
746 } SMARTLIST_FOREACH_END(ent);
758 if (most_recent == NULL)
760 const char *most_recent_sha3 =
762 LABEL_SHA3_DIGEST_UNCOMPRESSED);
763 if (BUG(most_recent_sha3 == NULL))
771 const char *this_diff_target_sha3 =
773 if (!this_diff_target_sha3)
775 if (strcmp(this_diff_target_sha3, most_recent_sha3)) {
779 } SMARTLIST_FOREACH_END(diff);
794 if (most_recent == NULL)
796 const char *most_recent_sha3_uncompressed =
798 LABEL_SHA3_DIGEST_UNCOMPRESSED);
802 if (BUG(most_recent_sha3_uncompressed == NULL))
805 const char *lv_sha3_uncompressed =
807 if (BUG(! lv_sha3_uncompressed))
809 if (!strcmp(lv_sha3_uncompressed, most_recent_sha3_uncompressed))
811 const char *lv_methodname =
813 if (! lv_methodname || strcmp(lv_methodname, retain_methodname)) {
817 } SMARTLIST_FOREACH_END(ent);
820 smartlist_free(objects);
821 smartlist_free(consensuses);
822 smartlist_free(diffs);
837 memcpy(&consdiff_cfg, cfg,
sizeof(consdiff_cfg));
873 }
else if (r == -2) {
901 } SMARTLIST_FOREACH_END(obj);
902 smartlist_free(objects);
915 strmap_t *have_diff_from = NULL;
928 LABEL_FLAVOR, flavname);
933 log_info(
LD_DIRSERV,
"No 'most recent' %s consensus found; "
934 "not making diffs", flavname);
940 const char *most_recent_valid_after =
942 if (BUG(most_recent_valid_after == NULL))
946 LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
953 LABEL_VALID_AFTER, most_recent_valid_after);
956 have_diff_from = strmap_new();
959 LABEL_FROM_VALID_AFTER);
962 strmap_set(have_diff_from, va, diff);
963 } SMARTLIST_FOREACH_END(diff);
972 if (strmap_get(have_diff_from, va) != NULL)
979 } SMARTLIST_FOREACH_END(ent);
982 "The most recent %s consensus is valid-after %s. We have diffs to "
983 "this consensus for %d/%d older %s consensuses. Generating diffs "
986 most_recent_valid_after,
987 smartlist_len(matches) - smartlist_len(compute_diffs_from),
988 smartlist_len(matches),
990 smartlist_len(compute_diffs_from));
998 if (BUG(c == most_recent))
1003 LABEL_SHA3_DIGEST_AS_SIGNED)<0) {
1009 this_sha3, most_recent_sha3)) {
1014 } SMARTLIST_FOREACH_END(c);
1017 smartlist_free(matches);
1018 smartlist_free(diffs);
1019 smartlist_free(compute_diffs_from);
1020 strmap_free(have_diff_from, NULL);
1035 LABEL_FLAVOR, flavname);
1040 const char *most_recent_sha3 =
1042 LABEL_SHA3_DIGEST_UNCOMPRESSED);
1043 if (BUG(most_recent_sha3 == NULL))
1051 const char *lv_compression =
1060 } SMARTLIST_FOREACH_END(ent);
1062 smartlist_free(matches);
1075 const char *lv_flavor =
1082 const char *lv_compression =
1085 if (lv_compression) {
1087 if (method == UNKNOWN_METHOD) {
1102 consensus_cache_entry_handle_new(diff));
1103 } SMARTLIST_FOREACH_END(diff);
1104 smartlist_free(diffs);
1162 const char *va1, *fva1, *va2, *fva2;
1203 if (n_to_remove <= 0) {
1216 if (++n_marked >= n_to_remove)
1218 } SMARTLIST_FOREACH_END(ent);
1219 smartlist_free(objects);
1230 if (BUG(n_marked < n_to_remove))
1249 } SMARTLIST_FOREACH_END(ent);
1250 smartlist_free(objects);
1260 for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
1262 next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
1263 cdm_diff_free(
this);
1307 const uint8_t *input,
size_t len,
1312 for (i = 0; i < n_methods; ++i) {
1317 if (0 ==
tor_compress(&result, &sz, (
const char*)input, len, method)) {
1318 results_out[i].
body = (uint8_t*)result;
1322 results_out[i].body,
1323 results_out[i].bodylen);
1325 LABEL_COMPRESSION_TYPE,
1348 const char *description)
1354 for (i = 0; i < n; ++i) {
1356 uint8_t *body_out = results[i].
body;
1357 size_t bodylen_out = results[i].
bodylen;
1360 if (body_out && bodylen_out && labels) {
1362 log_info(
LD_DIRSERV,
"Adding %s, compressed with %s",
1363 description, methodname);
1371 static ratelim_t cant_store_ratelim = RATELIM_INIT(5*60);
1373 "Unable to store object %s compressed with %s.",
1374 description, methodname);
1378 status = CDM_DIFF_PRESENT;
1379 handles_out[i] = consensus_cache_entry_handle_new(ent);
1419 const uint8_t *body;
1427 const char *lv_compression =
1435 if (method == NO_METHOD) {
1436 *out = (
const char *)body;
1440 rv =
tor_uncompress(owned_out, outlen, (
const char *)body, bodylen,
1456 const uint8_t *diff_from, *diff_to;
1457 size_t len_from, len_to;
1463 return WQ_RPL_REPLY;
1466 return WQ_RPL_REPLY;
1468 const char *lv_to_valid_after =
1470 const char *lv_to_fresh_until =
1472 const char *lv_to_valid_until =
1474 const char *lv_to_signatories =
1476 const char *lv_from_valid_after =
1478 const char *lv_from_digest =
1480 LABEL_SHA3_DIGEST_AS_SIGNED);
1481 const char *lv_from_flavor =
1483 const char *lv_to_flavor =
1485 const char *lv_to_digest =
1487 LABEL_SHA3_DIGEST_UNCOMPRESSED);
1489 if (! lv_from_digest) {
1494 return WQ_RPL_REPLY;
1498 if (BUG(!lv_to_valid_after) ||
1499 BUG(!lv_from_valid_after) ||
1500 BUG(!lv_from_flavor) ||
1501 BUG(!lv_to_flavor)) {
1502 return WQ_RPL_REPLY;
1505 if (BUG(strcmp(lv_from_flavor, lv_to_flavor))) {
1506 return WQ_RPL_REPLY;
1509 char *consensus_diff;
1511 const char *diff_from_nt = NULL, *diff_to_nt = NULL;
1512 char *owned1 = NULL, *owned2 = NULL;
1513 size_t diff_from_nt_len, diff_to_nt_len;
1517 return WQ_RPL_REPLY;
1522 return WQ_RPL_REPLY;
1537 if (!consensus_diff) {
1539 return WQ_RPL_REPLY;
1544 size_t difflen = strlen(consensus_diff);
1545 job->
out[0].
body = (uint8_t *) consensus_diff;
1549 if (lv_to_valid_until)
1551 if (lv_to_fresh_until)
1553 if (lv_to_signatories)
1556 LABEL_SHA3_DIGEST_UNCOMPRESSED,
1560 lv_from_valid_after);
1569 DOCTYPE_CONSENSUS_DIFF);
1580 (
const uint8_t*)consensus_diff, difflen, common_labels);
1582 config_free_lines(common_labels);
1583 return WQ_RPL_REPLY;
1586 #define consensus_diff_worker_job_free(job) \
1587 FREE_AND_NULL(consensus_diff_worker_job_t, \
1588 consensus_diff_worker_job_free_, (job))
1600 config_free_lines(job->
out[u].labels);
1621 const char *lv_from_digest =
1623 LABEL_SHA3_DIGEST_AS_SIGNED);
1624 const char *lv_to_digest =
1626 LABEL_SHA3_DIGEST_UNCOMPRESSED);
1627 const char *lv_flavor =
1629 if (BUG(lv_from_digest == NULL))
1630 lv_from_digest =
"???";
1631 if (BUG(lv_to_digest == NULL))
1632 lv_to_digest =
"???";
1639 LABEL_SHA3_DIGEST_AS_SIGNED) < 0))
1642 LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
1644 if (BUG(lv_flavor == NULL)) {
1651 memset(handles, 0,
sizeof(handles));
1653 char description[128];
1655 "consensus diff from %s to %s",
1656 lv_from_digest, lv_to_digest);
1664 if (status != CDM_DIFF_PRESENT) {
1667 "Worker was unable to compute consensus diff "
1668 "from %s to %s", lv_from_digest, lv_to_digest);
1670 status = CDM_DIFF_ERROR;
1677 consensus_cache_entry_handle_t *h = handles[u];
1678 int this_status = status;
1680 this_status = CDM_DIFF_ERROR;
1682 tor_assert_nonfatal(h != NULL || this_status == CDM_DIFF_ERROR);
1685 consensus_cache_entry_handle_free(handles[u]);
1689 consensus_diff_worker_job_free(job);
1710 const uint8_t *body;
1714 if (r1 < 0 || r2 < 0)
1727 consensus_diff_worker_job_free(job);
1736 size_t consensus_len;
1742 #define consensus_compress_worker_job_free(job) \
1743 FREE_AND_NULL(consensus_compress_worker_job_t, \
1744 consensus_compress_worker_job_free_, (job))
1755 config_free_lines(job->labels_in);
1758 config_free_lines(job->out[u].labels);
1773 const char *consensus = job->consensus;
1774 size_t bodylen = job->consensus_len;
1780 (
const uint8_t *)consensus, bodylen);
1782 const char *start, *end;
1784 &start, &end) < 0) {
1786 end = consensus+bodylen;
1789 (
const uint8_t *)start,
1798 (
const uint8_t*)consensus, bodylen, labels);
1799 config_free_lines(labels);
1800 return WQ_RPL_REPLY;
1813 consensus_cache_entry_handle_t *handles[
1815 memset(handles, 0,
sizeof(handles));
1828 if (handles[u] == NULL)
1834 consensus_compress_worker_job_free(job);
1848 size_t consensus_len,
1855 job->consensus = tor_memdup_nulterm(consensus, consensus_len);
1856 job->consensus_len = strlen(job->consensus);
1857 job->flavor = as_parsed->
flavor;
1859 char va_str[ISO_TIME_LEN+1];
1860 char vu_str[ISO_TIME_LEN+1];
1861 char fu_str[ISO_TIME_LEN+1];
1872 if (smartlist_len(vi->sigs) == 0)
1877 } SMARTLIST_FOREACH_END(vi);
1882 smartlist_free(hexvoters);
1892 consensus_compress_worker_job_free(job);