tor  0.4.2.0-alpha-dev
conscache.c
1 /* Copyright (c) 2017-2019, The Tor Project, Inc. */
2 /* See LICENSE for licensing information */
3 
4 #include "core/or/or.h"
5 
6 #include "app/config/config.h"
7 #include "feature/dircache/conscache.h"
9 #include "lib/fs/storagedir.h"
10 #include "lib/encoding/confline.h"
11 
12 #define CCE_MAGIC 0x17162253
13 
14 #ifdef _WIN32
15 /* On Windows, unlink won't work on a file if the file is actively mmap()ed.
16  * That forces us to be less aggressive about unlinking files, and causes other
17  * changes throughout our logic.
18  */
19 #define MUST_UNMAP_TO_UNLINK
20 #endif /* defined(_WIN32) */
21 
28  uint32_t magic;
29  HANDLE_ENTRY(consensus_cache_entry, consensus_cache_entry_t);
30  int32_t refcnt;
31  unsigned can_remove : 1;
33  unsigned release_aggressively : 1;
34 
36  char *fname;
42 
45  time_t unused_since;
49  size_t bodylen;
51  const uint8_t *body;
52 };
53 
62 
66  unsigned max_entries;
67 };
68 
69 static void consensus_cache_clear(consensus_cache_t *cache);
70 static void consensus_cache_rescan(consensus_cache_t *);
71 static void consensus_cache_entry_map(consensus_cache_t *,
73 static void consensus_cache_entry_unmap(consensus_cache_entry_t *ent);
74 
80 consensus_cache_open(const char *subdir, int max_entries)
81 {
82  int storagedir_max_entries;
83  consensus_cache_t *cache = tor_malloc_zero(sizeof(consensus_cache_t));
84  char *directory = get_cachedir_fname(subdir);
85  cache->max_entries = max_entries;
86 
87 #ifdef MUST_UNMAP_TO_UNLINK
88  /* If we can't unlink the files that we're still using, then we need to
89  * tell the storagedir backend to allow far more files than this consensus
90  * cache actually wants, so that it can hold files which, from this cache's
91  * perspective, have become useless.
92  */
93 #define VERY_LARGE_STORAGEDIR_LIMIT (1000*1000)
94  storagedir_max_entries = VERY_LARGE_STORAGEDIR_LIMIT;
95 #else /* !(defined(MUST_UNMAP_TO_UNLINK)) */
96  /* Otherwise, we can just tell the storagedir to use the same limits
97  * as this cache. */
98  storagedir_max_entries = max_entries;
99 #endif /* defined(MUST_UNMAP_TO_UNLINK) */
100 
101  cache->dir = storage_dir_new(directory, storagedir_max_entries);
102  tor_free(directory);
103  if (!cache->dir) {
104  tor_free(cache);
105  return NULL;
106  }
107 
108  consensus_cache_rescan(cache);
109  return cache;
110 }
111 
119 int
120 consensus_cache_may_overallocate(consensus_cache_t *cache)
121 {
122  (void) cache;
123 #ifdef MUST_UNMAP_TO_UNLINK
124  return 1;
125 #else
126  return 0;
127 #endif
128 }
129 
134 int
135 consensus_cache_register_with_sandbox(consensus_cache_t *cache,
136  struct sandbox_cfg_elem **cfg)
137 {
138 #ifdef MUST_UNMAP_TO_UNLINK
139  /* Our Linux sandbox doesn't support huge file lists like the one that would
140  * be generated by using VERY_LARGE_STORAGEDIR_LIMIT above in
141  * consensus_cache_open(). Since the Linux sandbox is the only one we have
142  * right now, we just assert that we never reach this point when we've had
143  * to use VERY_LARGE_STORAGEDIR_LIMIT.
144  *
145  * If at some point in the future we have a different sandbox mechanism that
146  * can handle huge file lists, we can remove this assertion or make it
147  * conditional.
148  */
149  tor_assert_nonfatal_unreached();
150 #endif /* defined(MUST_UNMAP_TO_UNLINK) */
151  return storage_dir_register_with_sandbox(cache->dir, cfg);
152 }
153 
158 static void
159 consensus_cache_clear(consensus_cache_t *cache)
160 {
161  consensus_cache_delete_pending(cache, 0);
162 
164  ent->in_cache = NULL;
165  consensus_cache_entry_decref(ent);
166  } SMARTLIST_FOREACH_END(ent);
167  smartlist_free(cache->entries);
168  cache->entries = NULL;
169 }
170 
174 void
175 consensus_cache_free_(consensus_cache_t *cache)
176 {
177  if (! cache)
178  return;
179 
180  if (cache->entries) {
181  consensus_cache_clear(cache);
182  }
183  storage_dir_free(cache->dir);
184  tor_free(cache);
185 }
186 
201 consensus_cache_add(consensus_cache_t *cache,
202  const config_line_t *labels,
203  const uint8_t *data,
204  size_t datalen)
205 {
206  char *fname = NULL;
207  int r = storage_dir_save_labeled_to_file(cache->dir,
208  labels, data, datalen, &fname);
209  if (r < 0 || fname == NULL) {
210  return NULL;
211  }
213  tor_malloc_zero(sizeof(consensus_cache_entry_t));
214  ent->magic = CCE_MAGIC;
215  ent->fname = fname;
216  ent->labels = config_lines_dup(labels);
217  ent->in_cache = cache;
218  ent->unused_since = TIME_MAX;
219  smartlist_add(cache->entries, ent);
220  /* Start the reference count at 2: the caller owns one copy, and the
221  * cache owns another.
222  */
223  ent->refcnt = 2;
224 
225  return ent;
226 }
227 
235 consensus_cache_find_first(consensus_cache_t *cache,
236  const char *key,
237  const char *value)
238 {
239  smartlist_t *tmp = smartlist_new();
240  consensus_cache_find_all(tmp, cache, key, value);
241  consensus_cache_entry_t *ent = NULL;
242  if (smartlist_len(tmp))
243  ent = smartlist_get(tmp, 0);
244  smartlist_free(tmp);
245  return ent;
246 }
247 
256 void
257 consensus_cache_find_all(smartlist_t *out,
258  consensus_cache_t *cache,
259  const char *key,
260  const char *value)
261 {
263  if (ent->can_remove == 1) {
264  /* We want to delete this; pretend it isn't there. */
265  continue;
266  }
267  if (! key) {
268  smartlist_add(out, ent);
269  continue;
270  }
271  const char *found_val = consensus_cache_entry_get_value(ent, key);
272  if (found_val && !strcmp(value, found_val)) {
273  smartlist_add(out, ent);
274  }
275  } SMARTLIST_FOREACH_END(ent);
276 }
277 
284 void
285 consensus_cache_filter_list(smartlist_t *lst,
286  const char *key,
287  const char *value)
288 {
289  if (BUG(lst == NULL))
290  return; // LCOV_EXCL_LINE
291  if (key == NULL)
292  return;
294  const char *found_val = consensus_cache_entry_get_value(ent, key);
295  if (! found_val || strcmp(value, found_val)) {
296  SMARTLIST_DEL_CURRENT(lst, ent);
297  }
298  } SMARTLIST_FOREACH_END(ent);
299 }
300 
308 const char *
309 consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
310  const char *key)
311 {
312  const config_line_t *match = config_line_find(ent->labels, key);
313  if (match)
314  return match->value;
315  else
316  return NULL;
317 }
318 
325 const config_line_t *
326 consensus_cache_entry_get_labels(const consensus_cache_entry_t *ent)
327 {
328  return ent->labels;
329 }
330 
334 void
335 consensus_cache_entry_incref(consensus_cache_entry_t *ent)
336 {
337  if (BUG(ent->magic != CCE_MAGIC))
338  return; // LCOV_EXCL_LINE
339  ++ent->refcnt;
340  ent->unused_since = TIME_MAX;
341 }
342 
349 void
350 consensus_cache_entry_decref(consensus_cache_entry_t *ent)
351 {
352  if (! ent)
353  return;
354  if (BUG(ent->refcnt <= 0))
355  return; // LCOV_EXCL_LINE
356  if (BUG(ent->magic != CCE_MAGIC))
357  return; // LCOV_EXCL_LINE
358 
359  --ent->refcnt;
360 
361  if (ent->refcnt == 1 && ent->in_cache) {
362  /* Only the cache has a reference: we don't need to keep the file
363  * mapped */
364  if (ent->map) {
365  if (ent->release_aggressively) {
366  consensus_cache_entry_unmap(ent);
367  } else {
368  ent->unused_since = approx_time();
369  }
370  }
371  return;
372  }
373 
374  if (ent->refcnt > 0)
375  return;
376 
377  /* Refcount is zero; we can free it. */
378  if (ent->map) {
379  consensus_cache_entry_unmap(ent);
380  }
381  tor_free(ent->fname);
382  config_free_lines(ent->labels);
383  consensus_cache_entry_handles_clear(ent);
384  memwipe(ent, 0, sizeof(consensus_cache_entry_t));
385  tor_free(ent);
386 }
387 
392 void
393 consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent)
394 {
395  ent->can_remove = 1;
396 }
397 
402 void
403 consensus_cache_entry_mark_for_aggressive_release(consensus_cache_entry_t *ent)
404 {
405  ent->release_aggressively = 1;
406 }
407 
416 int
417 consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
418  const uint8_t **body_out,
419  size_t *sz_out)
420 {
421  if (BUG(ent->magic != CCE_MAGIC))
422  return -1; // LCOV_EXCL_LINE
423 
424  if (! ent->map) {
425  if (! ent->in_cache)
426  return -1;
427 
428  consensus_cache_entry_map((consensus_cache_t *)ent->in_cache,
429  (consensus_cache_entry_t *)ent);
430  if (! ent->map) {
431  return -1;
432  }
433  }
434 
435  *body_out = ent->body;
436  *sz_out = ent->bodylen;
437  return 0;
438 }
439 
444 void
445 consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff)
446 {
448  tor_assert_nonfatal(ent->in_cache == cache);
449  if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
450  /* Somebody is using this entry right now */
451  continue;
452  }
453  if (ent->unused_since > cutoff) {
454  /* Has been unused only for a little while */
455  continue;
456  }
457  if (ent->map == NULL) {
458  /* Not actually mapped. */
459  continue;
460  }
461  consensus_cache_entry_unmap(ent);
462  } SMARTLIST_FOREACH_END(ent);
463 }
464 
468 int
469 consensus_cache_get_n_filenames_available(consensus_cache_t *cache)
470 {
471  tor_assert(cache);
472  int max = cache->max_entries;
473  int used = smartlist_len(storage_dir_list(cache->dir));
474 #ifdef MUST_UNMAP_TO_UNLINK
475  if (used > max)
476  return 0;
477 #else
478  tor_assert_nonfatal(max >= used);
479 #endif /* defined(MUST_UNMAP_TO_UNLINK) */
480  return max - used;
481 }
482 
488 void
489 consensus_cache_delete_pending(consensus_cache_t *cache, int force)
490 {
492  tor_assert_nonfatal(ent->in_cache == cache);
493  int force_ent = force;
494 #ifdef MUST_UNMAP_TO_UNLINK
495  /* We cannot delete anything with an active mmap on win32, so no
496  * force-deletion. */
497  if (ent->map) {
498  force_ent = 0;
499  }
500 #endif /* defined(MUST_UNMAP_TO_UNLINK) */
501  if (! force_ent) {
502  if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
503  /* Somebody is using this entry right now */
504  continue;
505  }
506  }
507  if (ent->can_remove == 0) {
508  /* Don't want to delete this. */
509  continue;
510  }
511  if (BUG(ent->refcnt <= 0)) {
512  continue; // LCOV_EXCL_LINE
513  }
514 
515  SMARTLIST_DEL_CURRENT(cache->entries, ent);
516  ent->in_cache = NULL;
517  char *fname = tor_strdup(ent->fname); /* save a copy */
518  consensus_cache_entry_decref(ent);
519  storage_dir_remove_file(cache->dir, fname);
520  tor_free(fname);
521  } SMARTLIST_FOREACH_END(ent);
522 }
523 
527 static void
528 consensus_cache_rescan(consensus_cache_t *cache)
529 {
530  if (cache->entries) {
531  consensus_cache_clear(cache);
532  }
533 
534  cache->entries = smartlist_new();
535  const smartlist_t *fnames = storage_dir_list(cache->dir);
536  SMARTLIST_FOREACH_BEGIN(fnames, const char *, fname) {
537  tor_mmap_t *map = NULL;
538  config_line_t *labels = NULL;
539  const uint8_t *body;
540  size_t bodylen;
541  map = storage_dir_map_labeled(cache->dir, fname,
542  &labels, &body, &bodylen);
543  if (! map) {
544  /* The ERANGE error might come from tor_mmap_file() -- it means the file
545  * was empty. EINVAL might come from ..map_labeled() -- it means the
546  * file was misformatted. In both cases, we should just delete it.
547  */
548  if (errno == ERANGE || errno == EINVAL) {
549  log_warn(LD_FS, "Found %s file %s in consensus cache; removing it.",
550  errno == ERANGE ? "empty" : "misformatted",
551  escaped(fname));
552  storage_dir_remove_file(cache->dir, fname);
553  } else {
554  /* Can't load this; continue */
555  log_warn(LD_FS, "Unable to map file %s from consensus cache: %s",
556  escaped(fname), strerror(errno));
557  }
558  continue;
559  }
561  tor_malloc_zero(sizeof(consensus_cache_entry_t));
562  ent->magic = CCE_MAGIC;
563  ent->fname = tor_strdup(fname);
564  ent->labels = labels;
565  ent->refcnt = 1;
566  ent->in_cache = cache;
567  ent->unused_since = TIME_MAX;
568  smartlist_add(cache->entries, ent);
569  tor_munmap_file(map); /* don't actually need to keep this around */
570  } SMARTLIST_FOREACH_END(fname);
571 }
572 
576 static void
577 consensus_cache_entry_map(consensus_cache_t *cache,
579 {
580  if (ent->map)
581  return;
582 
583  ent->map = storage_dir_map_labeled(cache->dir, ent->fname,
584  NULL, &ent->body, &ent->bodylen);
585  ent->unused_since = TIME_MAX;
586 }
587 
594 static void
595 consensus_cache_entry_unmap(consensus_cache_entry_t *ent)
596 {
597  ent->unused_since = TIME_MAX;
598  if (!ent->map)
599  return;
600 
601  tor_munmap_file(ent->map);
602  ent->map = NULL;
603  ent->body = NULL;
604  ent->bodylen = 0;
605  ent->unused_since = TIME_MAX;
606 }
607 
608 HANDLE_IMPL(consensus_cache_entry, consensus_cache_entry_t, )
609 
610 #ifdef TOR_UNIT_TESTS
611 
616 int
617 consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent)
618 {
619  if (ent->map) {
620  tor_assert(ent->body);
621  return 1;
622  } else {
623  tor_assert(!ent->body);
624  return 0;
625  }
626 }
627 #endif /* defined(TOR_UNIT_TESTS) */
const smartlist_t * storage_dir_list(storage_dir_t *d)
Definition: storagedir.c:179
char * fname
Definition: conscache.c:36
Header for confline.c.
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
unsigned release_aggressively
Definition: conscache.c:33
void smartlist_add(smartlist_t *sl, void *element)
int32_t refcnt
Definition: conscache.c:30
Header file for config.c.
unsigned can_remove
Definition: conscache.c:31
time_t unused_since
Definition: conscache.c:45
#define tor_free(p)
Definition: malloc.h:52
tor_mmap_t * storage_dir_map_labeled(storage_dir_t *dir, const char *fname, config_line_t **labels_out, const uint8_t **data_out, size_t *sz_out)
Definition: storagedir.c:399
const uint8_t * body
Definition: conscache.c:51
Definition: conscache.c:27
#define SMARTLIST_DEL_CURRENT(sl, var)
consensus_cache_t * in_cache
Definition: conscache.c:41
void memwipe(void *mem, uint8_t byte, size_t sz)
Definition: crypto_util.c:57
size_t bodylen
Definition: conscache.c:49
unsigned max_entries
Definition: conscache.c:66
Common functions for cryptographic routines.
tor_assert(buffer)
Master header file for Tor-specific functionality.
storage_dir_t * dir
Definition: conscache.c:59
int storage_dir_register_with_sandbox(storage_dir_t *d, sandbox_cfg_t **cfg)
Definition: storagedir.c:104
tor_mmap_t * map
Definition: conscache.c:47
smartlist_t * entries
Definition: conscache.c:61
#define LD_FS
Definition: log.h:68
int storage_dir_save_labeled_to_file(storage_dir_t *d, const config_line_t *labels, const uint8_t *data, size_t length, char **fname_out)
Definition: storagedir.c:345
config_line_t * config_lines_dup(const config_line_t *inp)
Definition: confline.c:227
const char * escaped(const char *s)
Definition: escape.c:126
const config_line_t * config_line_find(const config_line_t *lines, const char *key)
Definition: confline.c:74
Header for storagedir.c.
time_t approx_time(void)
Definition: approx_time.c:32
uint32_t magic
Definition: conscache.c:28
void storage_dir_remove_file(storage_dir_t *d, const char *fname)
Definition: storagedir.c:480
config_line_t * labels
Definition: conscache.c:39
storage_dir_t * storage_dir_new(const char *dirname, int max_files)
Definition: storagedir.c:68