Tor
0.4.7.0-alpha-dev
|
consensus diff manager functions More...
#include "core/or/or.h"
#include "app/config/config.h"
#include "feature/dircache/conscache.h"
#include "feature/dircommon/consdiff.h"
#include "feature/dircache/consdiffmgr.h"
#include "core/mainloop/cpuworker.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/dirparse/ns_parse.h"
#include "lib/evloop/compat_libevent.h"
#include "lib/evloop/workqueue.h"
#include "lib/compress/compress.h"
#include "lib/encoding/confline.h"
#include "feature/nodelist/networkstatus_st.h"
#include "feature/nodelist/networkstatus_voter_info_st.h"
Go to the source code of this file.
Data Structures | |
struct | cdm_diff_t |
struct | compressed_result_t |
struct | consensus_diff_worker_job_t |
struct | consensus_compress_worker_job_t |
Macros | |
#define | CONSDIFFMGR_PRIVATE |
#define | DOCTYPE_CONSENSUS "consensus" |
#define | DOCTYPE_CONSENSUS_DIFF "consensus-diff" |
#define | RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD |
#define | CACHE_MAX_NUM 128 |
#define | cdm_diff_free(diff) FREE_AND_NULL(cdm_diff_t, cdm_diff_free_, (diff)) |
#define | consensus_diff_worker_job_free(job) |
#define | consensus_compress_worker_job_free(job) |
#define | LABEL_DOCTYPE "document-type" |
#define | LABEL_VALID_AFTER "consensus-valid-after" |
#define | LABEL_FRESH_UNTIL "consensus-fresh-until" |
#define | LABEL_VALID_UNTIL "consensus-valid-until" |
#define | LABEL_SIGNATORIES "consensus-signatories" |
#define | LABEL_SHA3_DIGEST "sha3-digest" |
#define | LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed" |
#define | LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed" |
#define | LABEL_FLAVOR "consensus-flavor" |
#define | LABEL_FROM_SHA3_DIGEST "from-sha3-digest" |
#define | LABEL_TARGET_SHA3_DIGEST "target-sha3-digest" |
#define | LABEL_FROM_VALID_AFTER "from-valid-after" |
#define | LABEL_COMPRESSION_TYPE "compression" |
Enumerations | |
enum | cdm_diff_status_t { CDM_DIFF_PRESENT =1 , CDM_DIFF_IN_PROGRESS =2 , CDM_DIFF_ERROR =3 } |
Variables | |
static consensus_cache_t * | cons_diff_cache = NULL |
static int | cdm_cache_dirty = 0 |
static int | cdm_cache_loaded = 0 |
static const compress_method_t | compress_diffs_with [] |
static mainloop_event_t * | consdiffmgr_rescan_ev = NULL |
static const compress_method_t | compress_consensus_with [] |
static consensus_cache_entry_handle_t * | latest_consensus [N_CONSENSUS_FLAVORS][ARRAY_LENGTH(compress_consensus_with)] |
static int | background_compression = 0 |
consensus diff manager functions
This module is run by directory authorities and caches in order to remember a number of past consensus documents, and to generate and serve the diffs from those documents to the latest consensus.
Definition in file consdiffmgr.c.
#define consensus_compress_worker_job_free | ( | job | ) |
Definition at line 1752 of file consdiffmgr.c.
#define consensus_diff_worker_job_free | ( | job | ) |
Definition at line 1596 of file consdiffmgr.c.
#define LABEL_DOCTYPE "document-type" |
Labels to apply to items in the conscache object.
Definition at line 38 of file consdiffmgr.c.
#define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD |
For which compression method do we retain old consensuses? There's no need to keep all of them, since we won't be serving them. We'll go with ZLIB_METHOD because it's pretty fast and everyone has it.
Definition at line 145 of file consdiffmgr.c.
enum cdm_diff_status_t |
Possible status values for cdm_diff_t.cdm_diff_status
Definition at line 90 of file consdiffmgr.c.
STATIC consensus_cache_t* cdm_cache_get | ( | void | ) |
Helper: return the consensus_cache_t * that backs this manager, initializing it if needed.
Definition at line 411 of file consdiffmgr.c.
Referenced by consdiffmgr_configure(), consdiffmgr_consensus_load(), consdiffmgr_diffs_load(), consdiffmgr_ensure_space_for_files(), consdiffmgr_register_with_sandbox(), consdiffmgr_rescan_flavor_(), consdiffmgr_set_cache_flags(), and consdiffmgr_validate().
|
static |
Helper: initialize cons_diff_cache.
Definition at line 386 of file consdiffmgr.c.
STATIC consensus_cache_entry_t* cdm_cache_lookup_consensus | ( | consensus_flavor_t | flavor, |
time_t | valid_after | ||
) |
Helper: look for a consensus with the given flavor and valid_after time in the cache. Return that consensus if it's present, or NULL if it's missing.
Definition at line 469 of file consdiffmgr.c.
|
static |
Helper: compare two cdm_diff_t objects for key equality
Definition at line 224 of file consdiffmgr.c.
|
static |
Release all storage held in diff.
Definition at line 240 of file consdiffmgr.c.
|
static |
Helper: hash the key of a cdm_diff_t.
Definition at line 214 of file consdiffmgr.c.
|
static |
Examine the diff hashtable to see whether we know anything about computing a diff of type flav between consensuses with the two provided SHA3-256 digests. If a computation is in progress, or if the computation has already been tried and failed, return 1. Otherwise, note the computation as "in progress" so that we don't reattempt it later, and return 0.
Definition at line 274 of file consdiffmgr.c.
|
static |
Helper: Remove from the hash table every present (actually computed) diff of type flav whose target digest does not match unless_target_sha3_matches.
This function is used for the hash table to throw away references to diffs that do not lead to the most given consensus of a given flavor.
Definition at line 351 of file consdiffmgr.c.
|
static |
Update the status of the diff of type flav between consensuses with the two provided SHA3-256 digests, so that its status becomes status, and its value becomes the handle. If handle is NULL, then the old handle (if any) is freed, and replaced with NULL.
Definition at line 307 of file consdiffmgr.c.
|
static |
Create and return a new cdm_diff_t with the given values. Does not add it to the hashtable.
Definition at line 251 of file consdiffmgr.c.
STATIC int cdm_entry_get_sha3_value | ( | uint8_t * | digest_out, |
consensus_cache_entry_t * | ent, | ||
const char * | label | ||
) |
Helper: if there is a sha3-256 hex-encoded digest in ent with the given label, set digest_out to that value (decoded), and return 0.
Return -1 if there is no such label, and -2 if it is badly formatted.
Definition at line 445 of file consdiffmgr.c.
Referenced by consdiffmgr_validate().
|
static |
Helper: given a list of labels, prepend the hex-encoded SHA3 digest of the bodylen-byte object at body to those labels, with label as its label.
Definition at line 425 of file consdiffmgr.c.
Referenced by consensus_compress_worker_threadfn().
|
static |
Helper: compare two files by their from-valid-after and valid-after labels, trying to sort in ascending order by from-valid-after (when present) and valid-after (when not). Place everything that has neither label first in the list.
Definition at line 1168 of file consdiffmgr.c.
Referenced by consdiffmgr_ensure_space_for_files().
|
static |
Helper: used to sort two smartlists of consensus_cache_entry_t by their LABEL_VALID_AFTER labels.
Definition at line 584 of file consdiffmgr.c.
Referenced by sort_and_find_most_recent().
|
static |
Compress the bytestring input of length len using the n_methods compression methods listed in the array methods.
For each successful compression, set the fields in the results_out array in the position corresponding to the compression method. Use labels_in as a basis for the labels of the result.
Return 0 if all compression succeeded; -1 if any failed.
Definition at line 1315 of file consdiffmgr.c.
int consdiffmgr_add_consensus | ( | const char * | consensus, |
size_t | consensus_len, | ||
const networkstatus_t * | as_parsed | ||
) |
Given a buffer containing a networkstatus consensus, and the results of having parsed that consensus, add that consensus to the cache if it is not already present and not too old. Create new consensus diffs from or to that consensus as appropriate.
Return 0 on success and -1 on failure.
Definition at line 549 of file consdiffmgr.c.
int consdiffmgr_cleanup | ( | void | ) |
Perform periodic cleanup tasks on the consensus diff cache. Return the number of objects marked for deletion.
Definition at line 722 of file consdiffmgr.c.
Referenced by consdiffmgr_ensure_space_for_files(), and consdiffmgr_rescan().
void consdiffmgr_configure | ( | const consdiff_cfg_t * | cfg | ) |
Initialize the consensus diff manager and its cache, and configure its parameters based on the latest torrc and networkstatus parameters.
Definition at line 844 of file consdiffmgr.c.
|
static |
Scan the cache for the latest consensuses and add their handles to latest_consensus
Definition at line 1038 of file consdiffmgr.c.
Referenced by consdiffmgr_rescan().
|
static |
Scan the cache for diffs, and add them to the hashtable.
Definition at line 1079 of file consdiffmgr.c.
Referenced by consdiffmgr_rescan().
void consdiffmgr_enable_background_compression | ( | void | ) |
Tell the consdiffmgr backend to compress consensuses in worker threads.
Definition at line 1918 of file consdiffmgr.c.
|
static |
If there are not enough unused filenames to store n files, then delete old consensuses until there are. (We have to keep track of the number of filenames because of the way that the seccomp2 cache works.)
Return 0 on success, -1 on failure.
Definition at line 1195 of file consdiffmgr.c.
consdiff_status_t consdiffmgr_find_consensus | ( | struct consensus_cache_entry_t ** | entry_out, |
consensus_flavor_t | flavor, | ||
compress_method_t | method | ||
) |
If we know a consensus with the flavor flavor compressed with method, set *entry_out to that value. Return values are as for consdiffmgr_find_diff_from().
Definition at line 629 of file consdiffmgr.c.
consdiff_status_t consdiffmgr_find_diff_from | ( | consensus_cache_entry_t ** | entry_out, |
consensus_flavor_t | flavor, | ||
int | digest_type, | ||
const uint8_t * | digest, | ||
size_t | digestlen, | ||
compress_method_t | method | ||
) |
Look up consensus_cache_entry_t for the consensus of type flavor, from the source consensus with the specified digest (which must be SHA3).
If the diff is present, store it into *entry_out and return CONSDIFF_AVAILABLE. Otherwise return CONSDIFF_NOT_FOUND or CONSDIFF_IN_PROGRESS.
Definition at line 660 of file consdiffmgr.c.
void consdiffmgr_free_all | ( | void | ) |
Called before shutdown: drop all storage held by the consdiffmgr.c module.
Definition at line 1267 of file consdiffmgr.c.
int consdiffmgr_register_with_sandbox | ( | struct sandbox_cfg_elem_t ** | cfg | ) |
Tell the sandbox (if any) configured by cfg to allow the operations that the consensus diff manager will need.
Definition at line 857 of file consdiffmgr.c.
void consdiffmgr_rescan | ( | void | ) |
Build new diffs as needed.
Definition at line 1121 of file consdiffmgr.c.
Referenced by consdiffmgr_rescan_cb().
|
static |
Callback wrapper for consdiffmgr_rescan
Definition at line 1145 of file consdiffmgr.c.
|
static |
Helper: build new diffs of flavor as needed
Definition at line 920 of file consdiffmgr.c.
Referenced by consdiffmgr_rescan().
|
static |
Set consensus cache flags on the objects in this consdiffmgr.
Definition at line 1250 of file consdiffmgr.c.
int consdiffmgr_validate | ( | void | ) |
Scan the consensus diff manager's cache for any grossly malformed entries, and mark them as deletable. Return 0 if no problems were found; 1 if problems were found and fixed.
Definition at line 868 of file consdiffmgr.c.
int consensus_cache_entry_get_fresh_until | ( | const consensus_cache_entry_t * | ent, |
time_t * | out | ||
) |
Read the fresh-until time of cached object ent into *out and return 0, or return -1 if no such time was recorded.
Definition at line 1944 of file consdiffmgr.c.
int consensus_cache_entry_get_valid_after | ( | const consensus_cache_entry_t * | ent, |
time_t * | out | ||
) |
Read the valid after timestamp from the cached object ent into *out and return 0, or return -1 if no such time was recorded.
Definition at line 1977 of file consdiffmgr.c.
Referenced by spooled_resource_estimate_size().
int consensus_cache_entry_get_valid_until | ( | const consensus_cache_entry_t * | ent, |
time_t * | out | ||
) |
Read the valid until timestamp from the cached object ent into *out and return 0, or return -1 if no such time was recorded.
Definition at line 1960 of file consdiffmgr.c.
int consensus_cache_entry_get_voter_id_digests | ( | const consensus_cache_entry_t * | ent, |
smartlist_t * | out | ||
) |
Read the set of voters from the cached object ent into out, as a list of hex-encoded digests. Return 0 on success, -1 if no signatories were recorded.
Definition at line 1928 of file consdiffmgr.c.
Referenced by client_likes_consensus().
|
static |
Free all resources held in job
Definition at line 1760 of file consdiffmgr.c.
|
static |
Worker function: This function runs in the main thread, and receives a consensus_diff_compress_job_t that the worker thread has already processed.
Definition at line 1819 of file consdiffmgr.c.
|
static |
Worker function. This function runs inside a worker thread and receives a consensus_compress_worker_job_t as its input.
Definition at line 1778 of file consdiffmgr.c.
|
static |
Return i such that compress_consensus_with[i] == method. Return -1 if no such i exists.
Definition at line 612 of file consdiffmgr.c.
|
static |
Queue the job of computing the diff from diff_from to diff_to in a worker thread.
Definition at line 1707 of file consdiffmgr.c.
|
static |
Helper: release all storage held in job.
Definition at line 1604 of file consdiffmgr.c.
|
static |
Worker function: This function runs in the main thread, and receives a consensus_diff_worker_job_t that the worker thread has already processed.
Definition at line 1624 of file consdiffmgr.c.
|
static |
Worker function. This function runs inside a worker thread and receives a consensus_diff_worker_job_t as its input.
Definition at line 1462 of file consdiffmgr.c.
|
static |
Queue a job to compress consensus and store its compressed text in the cache.
Definition at line 1857 of file consdiffmgr.c.
|
static |
Return the maximum age (in seconds) of consensuses that we should consider storing. The available space in the directory may impose additional limits on how much we store.
Definition at line 498 of file consdiffmgr.c.
|
static |
Hashtable mapping flavor and source consensus digest to status. Configuration for this module
Definition at line 178 of file consdiffmgr.c.
|
static |
Mark the cache as dirty, and schedule a rescan event.
Definition at line 1154 of file consdiffmgr.c.
Referenced by consensus_compress_worker_replyfn().
STATIC unsigned n_consensus_compression_methods | ( | void | ) |
How many different methods will we try to use for diff compression?
Definition at line 136 of file consdiffmgr.c.
Referenced by consensus_compress_worker_replyfn().
STATIC unsigned n_diff_compression_methods | ( | void | ) |
How many different methods will we try to use for diff compression?
Definition at line 118 of file consdiffmgr.c.
Referenced by cdm_diff_ht_check_and_note_pending(), and consensus_diff_worker_job_free_().
|
static |
Helper: Sort lst by LABEL_VALID_AFTER and return the most recent entry.
Definition at line 599 of file consdiffmgr.c.
|
static |
Given an array of n compressed_result_t in results, as produced by compress_multiple, store them all into the consdiffmgr, and store handles to them in the handles_out array.
Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none was stored.
Definition at line 1354 of file consdiffmgr.c.
Referenced by consensus_compress_worker_replyfn().
STATIC int uncompress_or_set_ptr | ( | const char ** | out, |
size_t * | outlen, | ||
char ** | owned_out, | ||
consensus_cache_entry_t * | ent | ||
) |
Given a consensus_cache_entry_t, check whether it has a label claiming that it was compressed. If so, uncompress its contents into *out and set outlen to hold their size, and set *owned_out to a pointer that the caller will need to free. If not, just set *out and outlen to its extent in memory. Return 0 on success, -1 on failure.
Definition at line 1425 of file consdiffmgr.c.
|
static |
If true, we compress in worker threads.
Definition at line 1850 of file consdiffmgr.c.
Referenced by consdiffmgr_enable_background_compression().
|
static |
If true, we have learned at least one new consensus since the consensus cache was last up-to-date.
Definition at line 81 of file consdiffmgr.c.
Referenced by consdiffmgr_rescan(), and mark_cdm_cache_dirty().
|
static |
If true, we have scanned the cache to update our hashtable of diffs.
Definition at line 85 of file consdiffmgr.c.
Referenced by consdiffmgr_rescan().
|
static |
Which methods do we use for precompressing consensuses?
Definition at line 124 of file consdiffmgr.c.
Referenced by consensus_compress_worker_replyfn(), and n_consensus_compression_methods().
|
static |
Which methods do we use for precompressing diffs?
Definition at line 97 of file consdiffmgr.c.
Referenced by n_diff_compression_methods().
|
static |
Underlying directory that stores consensuses and consensus diffs. Don't use this directly: use cdm_cache_get() instead.
Definition at line 76 of file consdiffmgr.c.
Referenced by cdm_cache_init().
|
static |
Event for rescanning the cache.
Definition at line 111 of file consdiffmgr.c.
Referenced by mark_cdm_cache_dirty().
|
static |
Handles pointing to the latest consensus entries as compressed and stored.
Definition at line 150 of file consdiffmgr.c.