Line data Source code
1 : /* Copyright (c) 2017-2021, The Tor Project, Inc. */
2 : /* See LICENSE for licensing information */
3 :
4 : #define CONSDIFFMGR_PRIVATE
5 :
6 : #include "core/or/or.h"
7 : #include "app/config/config.h"
8 : #include "feature/dircache/conscache.h"
9 : #include "feature/dircommon/consdiff.h"
10 : #include "feature/dircache/consdiffmgr.h"
11 : #include "core/mainloop/cpuworker.h"
12 : #include "lib/crypt_ops/crypto_rand.h"
13 : #include "feature/nodelist/networkstatus.h"
14 : #include "feature/dirparse/ns_parse.h"
15 : #include "lib/evloop/workqueue.h"
16 : #include "lib/compress/compress.h"
17 : #include "lib/encoding/confline.h"
18 :
19 : #include "feature/nodelist/networkstatus_st.h"
20 :
21 : #include "test/test.h"
22 : #include "test/log_test_helpers.h"
23 :
24 : #define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm
25 :
26 : static char *
27 1 : consensus_diff_apply_(const char *c, const char *d)
28 : {
29 1 : size_t c_len = strlen(c);
30 1 : size_t d_len = strlen(d);
31 : // We use memdup here to ensure that the input is NOT nul-terminated.
32 : // This makes it likelier for us to spot bugs.
33 1 : char *c_tmp = tor_memdup(c, c_len);
34 1 : char *d_tmp = tor_memdup(d, d_len);
35 1 : char *result = consensus_diff_apply(c_tmp, c_len, d_tmp, d_len);
36 1 : tor_free(c_tmp);
37 1 : tor_free(d_tmp);
38 1 : return result;
39 : }
40 :
41 : // ============================== Setup/teardown the consdiffmgr
42 : // These functions get run before/after each test in this module
43 :
44 : static void *
45 11 : consdiffmgr_test_setup(const struct testcase_t *arg)
46 : {
47 11 : (void)arg;
48 11 : char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm"));
49 11 : tor_free(get_options_mutable()->CacheDirectory);
50 11 : get_options_mutable()->CacheDirectory = ddir_fname; // now owns the pointer.
51 11 : check_private_dir(ddir_fname, CPD_CREATE, NULL);
52 :
53 11 : consdiff_cfg_t consdiff_cfg = { 300 };
54 11 : consdiffmgr_configure(&consdiff_cfg);
55 11 : return (void *)1; // must return something non-null.
56 : }
57 : static int
58 11 : consdiffmgr_test_teardown(const struct testcase_t *arg, void *ignore)
59 : {
60 11 : (void)arg;
61 11 : (void)ignore;
62 11 : consdiffmgr_free_all();
63 11 : return 1;
64 : }
65 : static struct testcase_setup_t setup_diffmgr = {
66 : consdiffmgr_test_setup,
67 : consdiffmgr_test_teardown
68 : };
69 :
70 : // ============================== NS faking functions
71 : // These functions are for making quick fake consensus objects and
72 : // strings that are just good enough for consdiff and consdiffmgr.
73 :
74 : static networkstatus_t *
75 25 : fake_ns_new(consensus_flavor_t flav, time_t valid_after)
76 : {
77 25 : networkstatus_t *ns = tor_malloc_zero(sizeof(networkstatus_t));
78 25 : ns->type = NS_TYPE_CONSENSUS;
79 25 : ns->flavor = flav;
80 25 : ns->valid_after = valid_after;
81 25 : return ns;
82 : }
83 :
84 : static char *
85 22 : fake_ns_body_new(consensus_flavor_t flav, time_t valid_after)
86 : {
87 22 : const char *flavor_string = flav == FLAV_NS ? "" : " microdesc";
88 22 : char valid_after_string[ISO_TIME_LEN+1];
89 :
90 22 : format_iso_time(valid_after_string, valid_after);
91 22 : char *random_stuff = crypto_random_hostname(3, 25, "junk ", "");
92 22 : char *random_stuff2 = crypto_random_hostname(3, 10, "", "");
93 :
94 22 : char *consensus;
95 22 : tor_asprintf(&consensus,
96 : "network-status-version 3%s\n"
97 : "vote-status consensus\n"
98 : "valid-after %s\n"
99 : "r name ccccccccccccccccc etc\nsample\n"
100 : "r name eeeeeeeeeeeeeeeee etc\nbar\n"
101 : "%s\n"
102 : "directory-signature hello-there\n"
103 : "directory-signature %s\n",
104 : flavor_string,
105 : valid_after_string,
106 : random_stuff,
107 : random_stuff2);
108 22 : tor_free(random_stuff);
109 22 : tor_free(random_stuff2);
110 22 : return consensus;
111 : }
112 :
113 : // ============================== Cpuworker mocking code
114 : // These mocking functions and types capture the cpuworker calls
115 : // so we can inspect them and run them in the main thread.
116 : static smartlist_t *fake_cpuworker_queue = NULL;
117 : typedef struct fake_work_queue_ent_t {
118 : enum workqueue_reply_t (*fn)(void *, void *);
119 : void (*reply_fn)(void *);
120 : void *arg;
121 : } fake_work_queue_ent_t;
122 : static struct workqueue_entry_t *
123 15 : mock_cpuworker_queue_work(workqueue_priority_t prio,
124 : enum workqueue_reply_t (*fn)(void *, void *),
125 : void (*reply_fn)(void *),
126 : void *arg)
127 : {
128 15 : (void) prio;
129 :
130 15 : if (! fake_cpuworker_queue)
131 8 : fake_cpuworker_queue = smartlist_new();
132 :
133 15 : fake_work_queue_ent_t *ent = tor_malloc_zero(sizeof(*ent));
134 15 : ent->fn = fn;
135 15 : ent->reply_fn = reply_fn;
136 15 : ent->arg = arg;
137 15 : smartlist_add(fake_cpuworker_queue, ent);
138 15 : return (struct workqueue_entry_t *)ent;
139 : }
140 : static int
141 7 : mock_cpuworker_run_work(void)
142 : {
143 7 : if (! fake_cpuworker_queue)
144 : return 0;
145 19 : SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, {
146 : enum workqueue_reply_t r = ent->fn(NULL, ent->arg);
147 : if (r != WQ_RPL_REPLY)
148 : return -1;
149 : });
150 : return 0;
151 : }
152 : static void
153 7 : mock_cpuworker_handle_replies(void)
154 : {
155 7 : if (! fake_cpuworker_queue)
156 : return;
157 19 : SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, {
158 : ent->reply_fn(ent->arg);
159 : tor_free(ent);
160 : });
161 7 : smartlist_free(fake_cpuworker_queue);
162 7 : fake_cpuworker_queue = NULL;
163 : }
164 :
165 : // ============================== Other helpers
166 :
167 : static consdiff_status_t
168 29 : lookup_diff_from(consensus_cache_entry_t **out,
169 : consensus_flavor_t flav,
170 : const char *str1)
171 : {
172 29 : uint8_t digest[DIGEST256_LEN];
173 29 : if (router_get_networkstatus_v3_sha3_as_signed(digest,
174 : str1, strlen(str1))<0) {
175 0 : TT_FAIL(("Unable to compute sha3-as-signed"));
176 0 : return CONSDIFF_NOT_FOUND;
177 : }
178 29 : return consdiffmgr_find_diff_from(out, flav,
179 : DIGEST_SHA3_256, digest, sizeof(digest),
180 : NO_METHOD);
181 : }
182 :
183 : static int
184 18 : lookup_apply_and_verify_diff(consensus_flavor_t flav,
185 : const char *str1,
186 : const char *str2)
187 : {
188 18 : consensus_cache_entry_t *ent = NULL;
189 18 : consdiff_status_t status = lookup_diff_from(&ent, flav, str1);
190 18 : if (ent == NULL || status != CONSDIFF_AVAILABLE) {
191 : return -1;
192 : }
193 :
194 17 : consensus_cache_entry_incref(ent);
195 17 : size_t size;
196 17 : const char *diff_string = NULL;
197 17 : char *diff_owned = NULL;
198 17 : int r = uncompress_or_set_ptr(&diff_string, &size, &diff_owned, ent);
199 17 : consensus_cache_entry_decref(ent);
200 17 : if (diff_string == NULL || r < 0)
201 : return -1;
202 :
203 17 : char *applied = consensus_diff_apply(str1, strlen(str1), diff_string, size);
204 17 : tor_free(diff_owned);
205 17 : if (applied == NULL)
206 : return -1;
207 :
208 17 : int match = !strcmp(applied, str2);
209 17 : tor_free(applied);
210 17 : return match ? 0 : -1;
211 : }
212 :
213 : static void
214 2 : cdm_reload(void)
215 : {
216 2 : consdiffmgr_free_all();
217 2 : cdm_cache_get();
218 2 : consdiffmgr_rescan();
219 2 : }
220 :
221 : // ============================== Beginning of tests
222 :
223 : #if 0
224 : static int got_failure = 0;
225 : static void
226 : got_assertion_failure(void)
227 : {
228 : ++got_failure;
229 : }
230 :
231 : /* XXXX This test won't work, because there is currently no way to actually
232 : * XXXX capture a real assertion failure. */
233 : static void
234 : test_consdiffmgr_init_failure(void *arg)
235 : {
236 : (void)arg;
237 : // Capture assertions and bugs.
238 :
239 : /* As in ...test_setup, but do not create the datadir. The missing directory
240 : * will cause a failure. */
241 : char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm"));
242 : tor_free(get_options_mutable()->CacheDirectory);
243 : get_options_mutable()->CacheDirectory = ddir_fname; // now owns the pointer.
244 :
245 : consdiff_cfg_t consdiff_cfg = { 7200, 300 };
246 :
247 : tor_set_failed_assertion_callback(got_assertion_failure);
248 : tor_capture_bugs_(1);
249 : consdiffmgr_configure(&consdiff_cfg); // This should fail.
250 : tt_int_op(got_failure, OP_EQ, 1);
251 : const smartlist_t *bugs = tor_get_captured_bug_log_();
252 : tt_int_op(smartlist_len(bugs), OP_EQ, 1);
253 :
254 : done:
255 : tor_end_capture_bugs_();
256 : }
257 : #endif /* 0 */
258 :
259 : static void
260 1 : test_consdiffmgr_sha3_helper(void *arg)
261 : {
262 1 : (void) arg;
263 1 : consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
264 1 : config_line_t *lines = NULL;
265 1 : char *mem_op_hex_tmp = NULL;
266 1 : config_line_prepend(&lines, "good-sha",
267 : "F00DF00DF00DF00DF00DF00DF00DF00D"
268 : "F00DF00DF00DF00DF00DF00DF00DF00D");
269 1 : config_line_prepend(&lines, "short-sha",
270 : "F00DF00DF00DF00DF00DF00DF00DF00D"
271 : "F00DF00DF00DF00DF00DF00DF00DF0");
272 1 : config_line_prepend(&lines, "long-sha",
273 : "F00DF00DF00DF00DF00DF00DF00DF00D"
274 : "F00DF00DF00DF00DF00DF00DF00DF00DF00D");
275 1 : config_line_prepend(&lines, "not-sha",
276 : "F00DF00DF00DF00DF00DF00DF00DF00D"
277 : "F00DF00DF00DF00DF00DF00DF00DXXXX");
278 1 : consensus_cache_entry_t *ent =
279 1 : consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
280 :
281 1 : uint8_t buf[DIGEST256_LEN];
282 1 : tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, NULL, "good-sha"));
283 1 : tt_int_op(0, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "good-sha"));
284 1 : test_memeq_hex(buf, "F00DF00DF00DF00DF00DF00DF00DF00D"
285 : "F00DF00DF00DF00DF00DF00DF00DF00D");
286 :
287 1 : tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "missing-sha"));
288 1 : tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "short-sha"));
289 1 : tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "long-sha"));
290 1 : tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "not-sha"));
291 :
292 1 : done:
293 1 : consensus_cache_entry_decref(ent);
294 1 : config_free_lines(lines);
295 1 : tor_free(mem_op_hex_tmp);
296 1 : }
297 :
298 : static void
299 1 : test_consdiffmgr_add(void *arg)
300 : {
301 1 : (void) arg;
302 1 : time_t now = approx_time();
303 :
304 1 : const char *body = NULL;
305 1 : char *body_owned = NULL;
306 :
307 1 : consensus_cache_entry_t *ent = NULL;
308 1 : networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now);
309 1 : const char *dummy = "foo";
310 1 : int r = consdiffmgr_add_consensus(dummy, ns_tmp);
311 1 : tt_int_op(r, OP_EQ, 0);
312 :
313 : /* If we add it again, it won't work */
314 1 : setup_capture_of_logs(LOG_INFO);
315 1 : dummy = "bar";
316 1 : r = consdiffmgr_add_consensus(dummy, ns_tmp);
317 1 : tt_int_op(r, OP_EQ, -1);
318 1 : expect_single_log_msg_containing("We already have a copy of that "
319 : "consensus");
320 1 : mock_clean_saved_logs();
321 :
322 : /* But it will work fine if the flavor is different */
323 1 : dummy = "baz";
324 1 : ns_tmp->flavor = FLAV_MICRODESC;
325 1 : r = consdiffmgr_add_consensus(dummy, ns_tmp);
326 1 : tt_int_op(r, OP_EQ, 0);
327 :
328 : /* And it will work fine if the time is different */
329 1 : dummy = "quux";
330 1 : ns_tmp->flavor = FLAV_NS;
331 1 : ns_tmp->valid_after = now - 60;
332 1 : r = consdiffmgr_add_consensus(dummy, ns_tmp);
333 1 : tt_int_op(r, OP_EQ, 0);
334 :
335 : /* If we add one a long long time ago, it will fail. */
336 1 : dummy = "xyzzy";
337 1 : ns_tmp->valid_after = 86400 * 100; /* A few months into 1970 */
338 1 : r = consdiffmgr_add_consensus(dummy, ns_tmp);
339 1 : tt_int_op(r, OP_EQ, -1);
340 1 : expect_log_msg_containing("it's too old.");
341 :
342 : /* Try looking up a consensuses. */
343 1 : ent = cdm_cache_lookup_consensus(FLAV_NS, now-60);
344 1 : tt_assert(ent);
345 1 : consensus_cache_entry_incref(ent);
346 1 : size_t s;
347 1 : r = uncompress_or_set_ptr(&body, &s, &body_owned, ent);
348 1 : tt_int_op(r, OP_EQ, 0);
349 1 : tt_int_op(s, OP_EQ, 4);
350 1 : tt_mem_op(body, OP_EQ, "quux", 4);
351 :
352 : /* Try looking up another entry, but fail */
353 1 : tt_ptr_op(cdm_cache_lookup_consensus(FLAV_MICRODESC, now - 60), OP_EQ, NULL);
354 1 : tt_ptr_op(cdm_cache_lookup_consensus(FLAV_NS, now - 61), OP_EQ, NULL);
355 :
356 1 : done:
357 1 : networkstatus_vote_free(ns_tmp);
358 1 : teardown_capture_of_logs();
359 1 : consensus_cache_entry_decref(ent);
360 1 : tor_free(body_owned);
361 1 : }
362 :
363 : static void
364 1 : test_consdiffmgr_make_diffs(void *arg)
365 : {
366 1 : (void)arg;
367 1 : networkstatus_t *ns = NULL;
368 1 : char *ns_body = NULL, *md_ns_body = NULL, *md_ns_body_2 = NULL;
369 1 : char *applied = NULL, *diff_text = NULL;
370 1 : time_t now = approx_time();
371 1 : int r;
372 1 : consensus_cache_entry_t *diff = NULL;
373 1 : uint8_t md_ns_sha3[DIGEST256_LEN];
374 1 : consdiff_status_t diff_status;
375 :
376 1 : MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
377 :
378 : // Try rescan with no consensuses: shouldn't crash or queue work.
379 1 : consdiffmgr_rescan();
380 1 : tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
381 :
382 : // Make two consensuses, 1 hour sec ago.
383 1 : ns = fake_ns_new(FLAV_NS, now-3600);
384 1 : ns_body = fake_ns_body_new(FLAV_NS, now-3600);
385 1 : r = consdiffmgr_add_consensus(ns_body, ns);
386 1 : networkstatus_vote_free(ns);
387 1 : tor_free(ns_body);
388 1 : tt_int_op(r, OP_EQ, 0);
389 :
390 1 : ns = fake_ns_new(FLAV_MICRODESC, now-3600);
391 1 : md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600);
392 1 : r = consdiffmgr_add_consensus(md_ns_body, ns);
393 1 : router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body,
394 : strlen(md_ns_body));
395 1 : networkstatus_vote_free(ns);
396 1 : tt_int_op(r, OP_EQ, 0);
397 :
398 : // No diffs will be generated.
399 1 : consdiffmgr_rescan();
400 1 : tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
401 :
402 : // Add a MD consensus from 45 minutes ago. This should cause one diff
403 : // worth of work to get queued.
404 1 : ns = fake_ns_new(FLAV_MICRODESC, now-45*60);
405 1 : md_ns_body_2 = fake_ns_body_new(FLAV_MICRODESC, now-45*60);
406 1 : r = consdiffmgr_add_consensus(md_ns_body_2, ns);
407 1 : networkstatus_vote_free(ns);
408 1 : tt_int_op(r, OP_EQ, 0);
409 :
410 1 : consdiffmgr_rescan();
411 1 : tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
412 1 : tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
413 1 : diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC,
414 : DIGEST_SHA3_256,
415 : md_ns_sha3, DIGEST256_LEN,
416 : NO_METHOD);
417 1 : tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status);
418 :
419 : // Now run that process and get the diff.
420 1 : r = mock_cpuworker_run_work();
421 1 : tt_int_op(r, OP_EQ, 0);
422 1 : mock_cpuworker_handle_replies();
423 :
424 : // At this point we should be able to get that diff.
425 1 : diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC,
426 : DIGEST_SHA3_256,
427 : md_ns_sha3, DIGEST256_LEN,
428 : NO_METHOD);
429 1 : tt_int_op(CONSDIFF_AVAILABLE, OP_EQ, diff_status);
430 1 : tt_assert(diff);
431 :
432 : /* Make sure applying the diff actually works */
433 1 : const uint8_t *diff_body;
434 1 : size_t diff_size;
435 1 : r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size);
436 1 : tt_int_op(r, OP_EQ, 0);
437 1 : diff_text = tor_memdup_nulterm(diff_body, diff_size);
438 1 : applied = consensus_diff_apply_(md_ns_body, diff_text);
439 1 : tt_assert(applied);
440 1 : tt_str_op(applied, OP_EQ, md_ns_body_2);
441 :
442 : /* Rescan again: no more work to do. */
443 1 : consdiffmgr_rescan();
444 1 : tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
445 :
446 1 : done:
447 1 : tor_free(md_ns_body);
448 1 : tor_free(md_ns_body_2);
449 1 : tor_free(diff_text);
450 1 : tor_free(applied);
451 1 : }
452 :
453 : static void
454 1 : test_consdiffmgr_diff_rules(void *arg)
455 : {
456 1 : (void)arg;
457 : #define N 6
458 1 : char *md_body[N], *ns_body[N];
459 1 : networkstatus_t *md_ns[N], *ns_ns[N];
460 1 : int i;
461 :
462 1 : MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
463 :
464 : /* Create a bunch of consensus things at 15-second intervals. */
465 1 : time_t start = approx_time() - 120;
466 7 : for (i = 0; i < N; ++i) {
467 6 : time_t when = start + i * 15;
468 6 : md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
469 6 : ns_body[i] = fake_ns_body_new(FLAV_NS, when);
470 6 : md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
471 6 : ns_ns[i] = fake_ns_new(FLAV_NS, when);
472 : }
473 :
474 : /* For the MD consensuses: add 4 of them, and make sure that
475 : * diffs are created to one consensus (the most recent) only. */
476 1 : tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
477 1 : tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
478 1 : tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3]));
479 1 : tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[4], md_ns[4]));
480 1 : consdiffmgr_rescan();
481 1 : tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
482 1 : tt_int_op(3, OP_EQ, smartlist_len(fake_cpuworker_queue));
483 1 : tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
484 1 : mock_cpuworker_handle_replies();
485 1 : tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
486 :
487 : /* For the NS consensuses: add 3, generate, and add one older one and
488 : * make sure that older one is the only one whose diff is generated */
489 1 : tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[0], ns_ns[0]));
490 1 : tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[1], ns_ns[1]));
491 1 : tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[5], ns_ns[5]));
492 1 : consdiffmgr_rescan();
493 1 : tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
494 1 : tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
495 1 : tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
496 1 : mock_cpuworker_handle_replies();
497 :
498 : /* At this point, we should actually have working diffs! */
499 1 : tt_int_op(0, OP_EQ,
500 : lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5]));
501 1 : tt_int_op(0, OP_EQ,
502 : lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5]));
503 :
504 1 : tt_int_op(0, OP_EQ,
505 : lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4]));
506 1 : tt_int_op(0, OP_EQ,
507 : lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4]));
508 1 : tt_int_op(0, OP_EQ,
509 : lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4]));
510 :
511 : /* Self-to-self diff won't be present */
512 1 : consensus_cache_entry_t *ent;
513 1 : tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
514 : lookup_diff_from(&ent, FLAV_NS, ns_body[5]));
515 : /* No diff from 2 has been added yet */
516 1 : tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
517 : lookup_diff_from(&ent, FLAV_NS, ns_body[2]));
518 : /* No diff arriving at old things. */
519 1 : tt_int_op(-1, OP_EQ,
520 : lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
521 : /* No backwards diff */
522 1 : tt_int_op(-1, OP_EQ,
523 : lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[4], md_body[3]));
524 :
525 : /* Now, an update: add number 2 and make sure it's the only one whose diff
526 : * is regenerated. */
527 1 : tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[2], ns_ns[2]));
528 1 : consdiffmgr_rescan();
529 1 : tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
530 1 : tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
531 1 : tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
532 1 : mock_cpuworker_handle_replies();
533 :
534 1 : tt_int_op(0, OP_EQ,
535 : lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5]));
536 :
537 : /* Finally: reload, and make sure that the information is still indexed */
538 1 : cdm_reload();
539 :
540 1 : tt_int_op(0, OP_EQ,
541 : lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5]));
542 1 : tt_int_op(0, OP_EQ,
543 : lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5]));
544 1 : tt_int_op(0, OP_EQ,
545 : lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5]));
546 :
547 1 : tt_int_op(0, OP_EQ,
548 : lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4]));
549 1 : tt_int_op(0, OP_EQ,
550 : lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4]));
551 1 : tt_int_op(0, OP_EQ,
552 : lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4]));
553 :
554 1 : done:
555 7 : for (i = 0; i < N; ++i) {
556 6 : tor_free(md_body[i]);
557 6 : tor_free(ns_body[i]);
558 6 : networkstatus_vote_free(md_ns[i]);
559 6 : networkstatus_vote_free(ns_ns[i]);
560 : }
561 1 : UNMOCK(cpuworker_queue_work);
562 : #undef N
563 1 : }
564 :
565 : static void
566 1 : test_consdiffmgr_diff_failure(void *arg)
567 : {
568 1 : (void)arg;
569 1 : MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
570 :
571 : /* We're going to make sure that if we have a bogus request where
572 : * we can't actually compute a diff, the world must not end. */
573 1 : networkstatus_t *ns1 = NULL;
574 1 : networkstatus_t *ns2 = NULL;
575 1 : int r;
576 :
577 1 : ns1 = fake_ns_new(FLAV_NS, approx_time()-100);
578 1 : ns2 = fake_ns_new(FLAV_NS, approx_time()-50);
579 1 : r = consdiffmgr_add_consensus("foo bar baz\n", ns1);
580 1 : tt_int_op(r, OP_EQ, 0);
581 : // We refuse to compute a diff to or from a line holding only a single dot.
582 : // We can add it here, though.
583 1 : r = consdiffmgr_add_consensus("foo bar baz\n.\n.\n", ns2);
584 1 : tt_int_op(r, OP_EQ, 0);
585 :
586 1 : consdiffmgr_rescan();
587 1 : tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
588 1 : setup_capture_of_logs(LOG_WARN);
589 1 : tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
590 1 : tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
591 1 : expect_single_log_msg_containing("one of the lines to be added is \".\".");
592 1 : mock_clean_saved_logs();
593 1 : mock_cpuworker_handle_replies();
594 1 : expect_single_log_msg_containing("Worker was unable to compute consensus "
595 : "diff from ");
596 :
597 : /* Make sure the diff is not present */
598 1 : consensus_cache_entry_t *ent;
599 1 : tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
600 : lookup_diff_from(&ent, FLAV_NS, "foo bar baz\n"));
601 :
602 1 : done:
603 1 : teardown_capture_of_logs();
604 1 : UNMOCK(cpuworker_queue_work);
605 1 : networkstatus_vote_free(ns1);
606 1 : networkstatus_vote_free(ns2);
607 1 : }
608 :
609 : static void
610 1 : test_consdiffmgr_diff_pending(void *arg)
611 : {
612 : #define N 3
613 1 : (void)arg;
614 1 : char *md_body[N];
615 1 : networkstatus_t *md_ns[N];
616 1 : time_t start = approx_time() - 120;
617 1 : int i;
618 4 : for (i = 0; i < N; ++i) {
619 3 : time_t when = start + i * 30;
620 3 : md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
621 3 : md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
622 : }
623 :
624 1 : MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
625 :
626 1 : tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
627 1 : tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
628 : /* Make a diff */
629 1 : consdiffmgr_rescan();
630 1 : tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
631 :
632 : /* Look it up. Is it pending? */
633 1 : consensus_cache_entry_t *ent = NULL;
634 1 : consdiff_status_t diff_status;
635 1 : diff_status = lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]);
636 1 : tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status);
637 1 : tt_ptr_op(ent, OP_EQ, NULL);
638 :
639 : /* Add another old consensus. only one new diff should launch! */
640 1 : tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0]));
641 1 : consdiffmgr_rescan();
642 1 : tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
643 :
644 1 : tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
645 1 : mock_cpuworker_handle_replies();
646 :
647 1 : tt_int_op(0, OP_EQ,
648 : lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2]));
649 1 : tt_int_op(0, OP_EQ,
650 : lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
651 :
652 1 : done:
653 1 : UNMOCK(cpuworker_queue_work);
654 4 : for (i = 0; i < N; ++i) {
655 3 : tor_free(md_body[i]);
656 3 : networkstatus_vote_free(md_ns[i]);
657 : }
658 : #undef N
659 1 : }
660 :
661 : static void
662 1 : test_consdiffmgr_cleanup_old(void *arg)
663 : {
664 1 : (void)arg;
665 1 : config_line_t *labels = NULL;
666 1 : consensus_cache_entry_t *ent = NULL;
667 1 : consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
668 :
669 : /* This item will be will be cleanable because it has a valid-after
670 : * time far in the past. */
671 1 : config_line_prepend(&labels, "document-type", "confribble-blarg");
672 1 : config_line_prepend(&labels, "consensus-valid-after",
673 : "1980-10-10T10:10:10");
674 1 : ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
675 1 : tt_assert(ent);
676 1 : consensus_cache_entry_decref(ent);
677 :
678 1 : setup_capture_of_logs(LOG_DEBUG);
679 1 : tt_int_op(1, OP_EQ, consdiffmgr_cleanup());
680 1 : expect_log_msg_containing("Deleting entry because its consensus-valid-"
681 1 : "after value (1980-10-10T10:10:10) was too old");
682 :
683 1 : done:
684 1 : teardown_capture_of_logs();
685 1 : config_free_lines(labels);
686 1 : }
687 :
688 : static void
689 1 : test_consdiffmgr_cleanup_bad_valid_after(void *arg)
690 : {
691 : /* This will seem cleanable, but isn't, because its valid-after time is
692 : * malformed. */
693 :
694 1 : (void)arg;
695 1 : config_line_t *labels = NULL;
696 1 : consensus_cache_entry_t *ent = NULL;
697 1 : consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
698 :
699 1 : config_line_prepend(&labels, "document-type", "consensus");
700 1 : config_line_prepend(&labels, "consensus-valid-after",
701 : "whan that aprille with his shoures soote"); // (~1385?)
702 1 : ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
703 1 : tt_assert(ent);
704 1 : consensus_cache_entry_decref(ent);
705 :
706 1 : setup_capture_of_logs(LOG_DEBUG);
707 1 : tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
708 1 : expect_log_msg_containing("Ignoring entry because its consensus-valid-"
709 : "after value (\"whan that aprille with his "
710 1 : "shoures soote\") was unparseable");
711 :
712 1 : done:
713 1 : teardown_capture_of_logs();
714 1 : config_free_lines(labels);
715 1 : }
716 :
717 : static void
718 1 : test_consdiffmgr_cleanup_no_valid_after(void *arg)
719 : {
720 1 : (void)arg;
721 1 : config_line_t *labels = NULL;
722 1 : consensus_cache_entry_t *ent = NULL;
723 1 : consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
724 :
725 : /* This item will be will be uncleanable because it has no recognized
726 : * valid-after. */
727 1 : config_line_prepend(&labels, "document-type", "consensus");
728 1 : config_line_prepend(&labels, "confrooble-voolid-oofter",
729 : "2010-10-10T09:08:07");
730 1 : ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
731 1 : tt_assert(ent);
732 1 : consensus_cache_entry_decref(ent);
733 :
734 1 : setup_capture_of_logs(LOG_DEBUG);
735 1 : tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
736 1 : expect_log_msg_containing("Ignoring entry because it had no consensus-"
737 1 : "valid-after label");
738 :
739 1 : done:
740 1 : teardown_capture_of_logs();
741 1 : config_free_lines(labels);
742 1 : }
743 :
744 : static void
745 1 : test_consdiffmgr_cleanup_old_diffs(void *arg)
746 : {
747 1 : (void)arg;
748 : #define N 4
749 1 : char *md_body[N];
750 1 : networkstatus_t *md_ns[N];
751 1 : int i;
752 1 : consensus_cache_entry_t *hold_ent = NULL, *ent;
753 :
754 : /* Make sure that the cleanup function removes diffs to the not-most-recent
755 : * consensus. */
756 :
757 1 : MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
758 :
759 : /* Create a bunch of consensus things at 15-second intervals. */
760 1 : time_t start = approx_time() - 120;
761 5 : for (i = 0; i < N; ++i) {
762 4 : time_t when = start + i * 15;
763 4 : md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
764 4 : md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
765 : }
766 :
767 : /* add the first 3. */
768 1 : tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0]));
769 1 : tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
770 1 : tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
771 : /* Make diffs. */
772 1 : consdiffmgr_rescan();
773 1 : tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
774 1 : tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
775 1 : tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
776 1 : mock_cpuworker_handle_replies();
777 1 : tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
778 :
779 : /* Nothing is deletable now */
780 1 : tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
781 1 : tt_int_op(0, OP_EQ,
782 : lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2]));
783 1 : tt_int_op(0, OP_EQ,
784 : lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
785 :
786 1 : tt_int_op(CONSDIFF_AVAILABLE, OP_EQ,
787 : lookup_diff_from(&hold_ent, FLAV_MICRODESC, md_body[1]));
788 1 : consensus_cache_entry_incref(hold_ent); // incref, so it is preserved.
789 :
790 : /* Now add an even-more-recent consensus; this should make all previous
791 : * diffs deletable, and make delete */
792 1 : tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3]));
793 1 : tt_int_op(2 * n_diff_compression_methods() +
794 : (n_consensus_compression_methods() - 1) , OP_EQ,
795 : consdiffmgr_cleanup());
796 :
797 1 : tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
798 : lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0]));
799 : /* This one is marked deletable but still in the hashtable */
800 1 : tt_int_op(CONSDIFF_AVAILABLE, OP_EQ,
801 : lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]));
802 1 : tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
803 : lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2]));
804 :
805 : /* Everything should be valid at this point */
806 1 : tt_int_op(0, OP_EQ, consdiffmgr_validate());
807 :
808 : /* And if we recan NOW, we'll purge the hashtable of the entries,
809 : * and launch attempts to generate new ones */
810 1 : consdiffmgr_rescan();
811 1 : tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
812 : lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0]));
813 1 : tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
814 : lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]));
815 1 : tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
816 : lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2]));
817 :
818 : /* We're still holding on to this, though, so we can still map it! */
819 1 : const uint8_t *t1 = NULL;
820 1 : size_t s;
821 1 : int r = consensus_cache_entry_get_body(hold_ent, &t1, &s);
822 1 : tt_int_op(r, OP_EQ, 0);
823 1 : tt_assert(t1);
824 :
825 1 : done:
826 5 : for (i = 0; i < N; ++i) {
827 4 : tor_free(md_body[i]);
828 4 : networkstatus_vote_free(md_ns[i]);
829 : }
830 1 : consensus_cache_entry_decref(hold_ent);
831 1 : UNMOCK(cpuworker_queue_work);
832 : #undef N
833 1 : }
834 :
835 : static void
836 1 : test_consdiffmgr_validate(void *arg)
837 : {
838 1 : (void)arg;
839 1 : config_line_t *lines = NULL;
840 1 : consensus_cache_entry_t *ent = NULL;
841 1 : consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
842 1 : smartlist_t *vals = smartlist_new();
843 :
844 : /* Put these: objects in the cache: one with a good sha3, one with bad sha3,
845 : * one with a wrong sha3, and one with no sha3. */
846 1 : config_line_prepend(&lines, "id", "wrong sha3");
847 1 : config_line_prepend(&lines, "sha3-digest",
848 : "F00DF00DF00DF00DF00DF00DF00DF00D"
849 : "F00DF00DF00DF00DF00DF00DF00DF00D");
850 1 : ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
851 1 : consensus_cache_entry_decref(ent);
852 1 : config_free_lines(lines);
853 1 : lines = NULL;
854 :
855 1 : config_line_prepend(&lines, "id", "bad sha3");
856 1 : config_line_prepend(&lines, "sha3-digest",
857 : "now is the winter of our dicotheque");
858 1 : ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
859 1 : consensus_cache_entry_decref(ent);
860 1 : config_free_lines(lines);
861 1 : lines = NULL;
862 :
863 1 : config_line_prepend(&lines, "id", "no sha3");
864 1 : ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
865 1 : consensus_cache_entry_decref(ent);
866 1 : config_free_lines(lines);
867 1 : lines = NULL;
868 :
869 1 : config_line_prepend(&lines, "id", "good sha3");
870 1 : config_line_prepend(&lines, "sha3-digest",
871 : "8d8b1998616cd6b4c4055da8d38728dc"
872 : "93c758d4131a53c7d81aa6337dee1c05");
873 1 : ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
874 1 : consensus_cache_entry_decref(ent);
875 1 : config_free_lines(lines);
876 1 : lines = NULL;
877 :
878 1 : cdm_reload();
879 1 : cache = cdm_cache_get();
880 1 : tt_int_op(1, OP_EQ, consdiffmgr_validate());
881 :
882 1 : consensus_cache_find_all(vals, cache, "id", "good sha3");
883 1 : tt_int_op(smartlist_len(vals), OP_EQ, 1);
884 1 : smartlist_clear(vals);
885 :
886 1 : consensus_cache_find_all(vals, cache, "id", "no sha3");
887 1 : tt_int_op(smartlist_len(vals), OP_EQ, 1);
888 1 : smartlist_clear(vals);
889 :
890 1 : consensus_cache_find_all(vals, cache, "id", "wrong sha3");
891 1 : tt_int_op(smartlist_len(vals), OP_EQ, 0);
892 1 : consensus_cache_find_all(vals, cache, "id", "bad sha3");
893 1 : tt_int_op(smartlist_len(vals), OP_EQ, 0);
894 :
895 1 : done:
896 1 : smartlist_free(vals);
897 1 : }
898 :
899 : #define TEST(name) \
900 : { #name, test_consdiffmgr_ ## name , TT_FORK, &setup_diffmgr, NULL }
901 :
902 : struct testcase_t consdiffmgr_tests[] = {
903 : #if 0
904 : { "init_failure", test_consdiffmgr_init_failure, TT_FORK, NULL, NULL },
905 : #endif
906 : TEST(sha3_helper),
907 : TEST(add),
908 : TEST(make_diffs),
909 : TEST(diff_rules),
910 : TEST(diff_failure),
911 : TEST(diff_pending),
912 : TEST(cleanup_old),
913 : TEST(cleanup_bad_valid_after),
914 : TEST(cleanup_no_valid_after),
915 : TEST(cleanup_old_diffs),
916 : TEST(validate),
917 :
918 : // XXXX Test: non-cacheing cases of replyfn().
919 :
920 : END_OF_TESTCASES
921 : };
|
