tor  0.4.2.1-alpha-dev
unparseable.c
1 /* Copyright (c) 2001 Matej Pfajfar.
2  * Copyright (c) 2001-2004, Roger Dingledine.
3  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
4  * Copyright (c) 2007-2019, The Tor Project, Inc. */
5 /* See LICENSE for licensing information */
6 
7 #define UNPARSEABLE_PRIVATE
8 
9 #include "core/or/or.h"
10 #include "app/config/config.h"
12 #include "lib/sandbox/sandbox.h"
13 
14 #ifdef HAVE_SYS_STAT_H
15 #include <sys/stat.h>
16 #endif
17 
18 /* Dump mechanism for unparseable descriptors */
19 
21 STATIC smartlist_t *descs_dumped = NULL;
23 STATIC uint64_t len_descs_dumped = 0;
25 static int have_dump_desc_dir = 0;
26 static int problem_with_dump_desc_dir = 0;
27 
28 #define DESC_DUMP_DATADIR_SUBDIR "unparseable-descs"
29 #define DESC_DUMP_BASE_FILENAME "unparseable-desc"
30 
32 void
34 {
35  char *dump_desc_dir;
36 
37  dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR);
38 
39  /*
40  * We just check for it, don't create it at this point; we'll
41  * create it when we need it if it isn't already there.
42  */
43  if (check_private_dir(dump_desc_dir, CPD_CHECK, get_options()->User) < 0) {
44  /* Error, log and flag it as having a problem */
45  log_notice(LD_DIR,
46  "Doesn't look like we'll be able to create descriptor dump "
47  "directory %s; dumps will be disabled.",
48  dump_desc_dir);
49  problem_with_dump_desc_dir = 1;
50  tor_free(dump_desc_dir);
51  return;
52  }
53 
54  /* Check if it exists */
55  switch (file_status(dump_desc_dir)) {
56  case FN_DIR:
57  /* We already have a directory */
58  have_dump_desc_dir = 1;
59  break;
60  case FN_NOENT:
61  /* Nothing, we'll need to create it later */
62  have_dump_desc_dir = 0;
63  break;
64  case FN_ERROR:
65  /* Log and flag having a problem */
66  log_notice(LD_DIR,
67  "Couldn't check whether descriptor dump directory %s already"
68  " exists: %s",
69  dump_desc_dir, strerror(errno));
70  problem_with_dump_desc_dir = 1;
71  break;
72  case FN_FILE:
73  case FN_EMPTY:
74  default:
75  /* Something else was here! */
76  log_notice(LD_DIR,
77  "Descriptor dump directory %s already exists and isn't a "
78  "directory",
79  dump_desc_dir);
80  problem_with_dump_desc_dir = 1;
81  }
82 
83  if (have_dump_desc_dir && !problem_with_dump_desc_dir) {
84  dump_desc_populate_fifo_from_directory(dump_desc_dir);
85  }
86 
87  tor_free(dump_desc_dir);
88 }
89 
91 static void
92 dump_desc_create_dir(void)
93 {
94  char *dump_desc_dir;
95 
96  /* If the problem flag is set, skip it */
97  if (problem_with_dump_desc_dir) return;
98 
99  /* Do we need it? */
100  if (!have_dump_desc_dir) {
101  dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR);
102 
103  if (check_private_dir(dump_desc_dir, CPD_CREATE,
104  get_options()->User) < 0) {
105  log_notice(LD_DIR,
106  "Failed to create descriptor dump directory %s",
107  dump_desc_dir);
108  problem_with_dump_desc_dir = 1;
109  }
110 
111  /* Okay, we created it */
112  have_dump_desc_dir = 1;
113 
114  tor_free(dump_desc_dir);
115  }
116 }
117 
124 static void
125 dump_desc_fifo_add_and_clean(char *filename, const uint8_t *digest_sha256,
126  size_t len)
127 {
128  dumped_desc_t *ent = NULL, *tmp;
129  uint64_t max_len;
130 
131  tor_assert(filename != NULL);
132  tor_assert(digest_sha256 != NULL);
133 
134  if (descs_dumped == NULL) {
135  /* We better have no length, then */
136  tor_assert(len_descs_dumped == 0);
137  /* Make a smartlist */
138  descs_dumped = smartlist_new();
139  }
140 
141  /* Make a new entry to put this one in */
142  ent = tor_malloc_zero(sizeof(*ent));
143  ent->filename = filename;
144  ent->len = len;
145  ent->when = time(NULL);
146  memcpy(ent->digest_sha256, digest_sha256, DIGEST256_LEN);
147 
148  /* Do we need to do some cleanup? */
149  max_len = get_options()->MaxUnparseableDescSizeToLog;
150  /* Iterate over the list until we've freed enough space */
151  while (len > max_len - len_descs_dumped &&
152  smartlist_len(descs_dumped) > 0) {
153  /* Get the oldest thing on the list */
154  tmp = (dumped_desc_t *)(smartlist_get(descs_dumped, 0));
155 
156  /*
157  * Check if it matches the filename we just added, so we don't delete
158  * something we just emitted if we get repeated identical descriptors.
159  */
160  if (strcmp(tmp->filename, filename) != 0) {
161  /* Delete it and adjust the length counter */
162  tor_unlink(tmp->filename);
163  tor_assert(len_descs_dumped >= tmp->len);
164  len_descs_dumped -= tmp->len;
165  log_info(LD_DIR,
166  "Deleting old unparseable descriptor dump %s due to "
167  "space limits",
168  tmp->filename);
169  } else {
170  /*
171  * Don't delete, but do adjust the counter since we will bump it
172  * later
173  */
174  tor_assert(len_descs_dumped >= tmp->len);
175  len_descs_dumped -= tmp->len;
176  log_info(LD_DIR,
177  "Replacing old descriptor dump %s with new identical one",
178  tmp->filename);
179  }
180 
181  /* Free it and remove it from the list */
182  smartlist_del_keeporder(descs_dumped, 0);
183  tor_free(tmp->filename);
184  tor_free(tmp);
185  }
186 
187  /* Append our entry to the end of the list and bump the counter */
188  smartlist_add(descs_dumped, ent);
189  len_descs_dumped += len;
190 }
191 
195 static int
196 dump_desc_fifo_bump_hash(const uint8_t *digest_sha256)
197 {
198  dumped_desc_t *match = NULL;
199 
200  tor_assert(digest_sha256);
201 
202  if (descs_dumped) {
203  /* Find a match if one exists */
204  SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) {
205  if (ent &&
206  tor_memeq(ent->digest_sha256, digest_sha256, DIGEST256_LEN)) {
207  /*
208  * Save a pointer to the match and remove it from its current
209  * position.
210  */
211  match = ent;
212  SMARTLIST_DEL_CURRENT_KEEPORDER(descs_dumped, ent);
213  break;
214  }
215  } SMARTLIST_FOREACH_END(ent);
216 
217  if (match) {
218  /* Update the timestamp */
219  match->when = time(NULL);
220  /* Add it back at the end of the list */
221  smartlist_add(descs_dumped, match);
222 
223  /* Indicate we found one */
224  return 1;
225  }
226  }
227 
228  return 0;
229 }
230 
233 void
235 {
236  if (descs_dumped) {
237  /* Free each descriptor */
238  SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) {
239  tor_assert(ent);
240  tor_free(ent->filename);
241  tor_free(ent);
242  } SMARTLIST_FOREACH_END(ent);
243  /* Free the list */
244  smartlist_free(descs_dumped);
245  descs_dumped = NULL;
246  len_descs_dumped = 0;
247  }
248 }
249 
254 MOCK_IMPL(STATIC dumped_desc_t *,
255 dump_desc_populate_one_file, (const char *dirname, const char *f))
256 {
257  dumped_desc_t *ent = NULL;
258  char *path = NULL, *desc = NULL;
259  const char *digest_str;
260  char digest[DIGEST256_LEN], content_digest[DIGEST256_LEN];
261  /* Expected prefix before digest in filenames */
262  const char *f_pfx = DESC_DUMP_BASE_FILENAME ".";
263  /*
264  * Stat while reading; this is important in case the file
265  * contains a NUL character.
266  */
267  struct stat st;
268 
269  /* Sanity-check args */
270  tor_assert(dirname != NULL);
271  tor_assert(f != NULL);
272 
273  /* Form the full path */
274  tor_asprintf(&path, "%s" PATH_SEPARATOR "%s", dirname, f);
275 
276  /* Check that f has the form DESC_DUMP_BASE_FILENAME.<digest256> */
277 
278  if (!strcmpstart(f, f_pfx)) {
279  /* It matches the form, but is the digest parseable as such? */
280  digest_str = f + strlen(f_pfx);
281  if (base16_decode(digest, DIGEST256_LEN,
282  digest_str, strlen(digest_str)) != DIGEST256_LEN) {
283  /* We failed to decode it */
284  digest_str = NULL;
285  }
286  } else {
287  /* No match */
288  digest_str = NULL;
289  }
290 
291  if (!digest_str) {
292  /* We couldn't get a sensible digest */
293  log_notice(LD_DIR,
294  "Removing unrecognized filename %s from unparseable "
295  "descriptors directory", f);
296  tor_unlink(path);
297  /* We're done */
298  goto done;
299  }
300 
301  /*
302  * The filename has the form DESC_DUMP_BASE_FILENAME "." <digest256> and
303  * we've decoded the digest. Next, check that we can read it and the
304  * content matches this digest. We are relying on the fact that if the
305  * file contains a '\0', read_file_to_str() will allocate space for and
306  * read the entire file and return the correct size in st.
307  */
308  desc = read_file_to_str(path, RFTS_IGNORE_MISSING|RFTS_BIN, &st);
309  if (!desc) {
310  /* We couldn't read it */
311  log_notice(LD_DIR,
312  "Failed to read %s from unparseable descriptors directory; "
313  "attempting to remove it.", f);
314  tor_unlink(path);
315  /* We're done */
316  goto done;
317  }
318 
319 #if SIZE_MAX > UINT64_MAX
320  if (BUG((uint64_t)st.st_size > (uint64_t)SIZE_MAX)) {
321  /* LCOV_EXCL_START
322  * Should be impossible since RFTS above should have failed to read the
323  * huge file into RAM. */
324  goto done;
325  /* LCOV_EXCL_STOP */
326  }
327 #endif /* SIZE_MAX > UINT64_MAX */
328  if (BUG(st.st_size < 0)) {
329  /* LCOV_EXCL_START
330  * Should be impossible, since the OS isn't supposed to be b0rken. */
331  goto done;
332  /* LCOV_EXCL_STOP */
333  }
334  /* (Now we can be sure that st.st_size is safe to cast to a size_t.) */
335 
336  /*
337  * We got one; now compute its digest and check that it matches the
338  * filename.
339  */
340  if (crypto_digest256((char *)content_digest, desc, (size_t) st.st_size,
341  DIGEST_SHA256) < 0) {
342  /* Weird, but okay */
343  log_info(LD_DIR,
344  "Unable to hash content of %s from unparseable descriptors "
345  "directory", f);
346  tor_unlink(path);
347  /* We're done */
348  goto done;
349  }
350 
351  /* Compare the digests */
352  if (tor_memneq(digest, content_digest, DIGEST256_LEN)) {
353  /* No match */
354  log_info(LD_DIR,
355  "Hash of %s from unparseable descriptors directory didn't "
356  "match its filename; removing it", f);
357  tor_unlink(path);
358  /* We're done */
359  goto done;
360  }
361 
362  /* Okay, it's a match, we should prepare ent */
363  ent = tor_malloc_zero(sizeof(dumped_desc_t));
364  ent->filename = path;
365  memcpy(ent->digest_sha256, digest, DIGEST256_LEN);
366  ent->len = (size_t) st.st_size;
367  ent->when = st.st_mtime;
368  /* Null out path so we don't free it out from under ent */
369  path = NULL;
370 
371  done:
372  /* Free allocations if we had them */
373  tor_free(desc);
374  tor_free(path);
375 
376  return ent;
377 }
378 
383 static int
384 dump_desc_compare_fifo_entries(const void **a_v, const void **b_v)
385 {
386  const dumped_desc_t **a = (const dumped_desc_t **)a_v;
387  const dumped_desc_t **b = (const dumped_desc_t **)b_v;
388 
389  if ((a != NULL) && (*a != NULL)) {
390  if ((b != NULL) && (*b != NULL)) {
391  /* We have sensible dumped_desc_ts to compare */
392  if ((*a)->when < (*b)->when) {
393  return -1;
394  } else if ((*a)->when == (*b)->when) {
395  return 0;
396  } else {
397  return 1;
398  }
399  } else {
400  /*
401  * We shouldn't see this, but what the hell, NULLs precede everythin
402  * else
403  */
404  return 1;
405  }
406  } else {
407  return -1;
408  }
409 }
410 
420 STATIC void
421 dump_desc_populate_fifo_from_directory(const char *dirname)
422 {
423  smartlist_t *files = NULL;
424  dumped_desc_t *ent = NULL;
425 
426  tor_assert(dirname != NULL);
427 
428  /* Get a list of files */
429  files = tor_listdir(dirname);
430  if (!files) {
431  log_notice(LD_DIR,
432  "Unable to get contents of unparseable descriptor dump "
433  "directory %s",
434  dirname);
435  return;
436  }
437 
438  /*
439  * Iterate through the list and decide which files should go in the
440  * FIFO and which should be purged.
441  */
442 
443  SMARTLIST_FOREACH_BEGIN(files, char *, f) {
444  /* Try to get a FIFO entry */
445  ent = dump_desc_populate_one_file(dirname, f);
446  if (ent) {
447  /*
448  * We got one; add it to the FIFO. No need for duplicate checking
449  * here since we just verified the name and digest match.
450  */
451 
452  /* Make sure we have a list to add it to */
453  if (!descs_dumped) {
454  descs_dumped = smartlist_new();
455  len_descs_dumped = 0;
456  }
457 
458  /* Add it and adjust the counter */
459  smartlist_add(descs_dumped, ent);
460  len_descs_dumped += ent->len;
461  }
462  /*
463  * If we didn't, we will have unlinked the file if necessary and
464  * possible, and emitted a log message about it, so just go on to
465  * the next.
466  */
467  } SMARTLIST_FOREACH_END(f);
468 
469  /* Did we get anything? */
470  if (descs_dumped != NULL) {
471  /* Sort the FIFO in order of increasing timestamp */
472  smartlist_sort(descs_dumped, dump_desc_compare_fifo_entries);
473 
474  /* Log some stats */
475  log_info(LD_DIR,
476  "Reloaded unparseable descriptor dump FIFO with %d dump(s) "
477  "totaling %"PRIu64 " bytes",
478  smartlist_len(descs_dumped), (len_descs_dumped));
479  }
480 
481  /* Free the original list */
482  SMARTLIST_FOREACH(files, char *, f, tor_free(f));
483  smartlist_free(files);
484 }
485 
490 MOCK_IMPL(void,
491 dump_desc,(const char *desc, const char *type))
492 {
493  tor_assert(desc);
494  tor_assert(type);
495  size_t len;
496  /* The SHA256 of the string */
497  uint8_t digest_sha256[DIGEST256_LEN];
498  char digest_sha256_hex[HEX_DIGEST256_LEN+1];
499  /* Filename to log it to */
500  char *debugfile, *debugfile_base;
501 
502  /* Get the hash for logging purposes anyway */
503  len = strlen(desc);
504  if (crypto_digest256((char *)digest_sha256, desc, len,
505  DIGEST_SHA256) < 0) {
506  log_info(LD_DIR,
507  "Unable to parse descriptor of type %s, and unable to even hash"
508  " it!", type);
509  goto err;
510  }
511 
512  base16_encode(digest_sha256_hex, sizeof(digest_sha256_hex),
513  (const char *)digest_sha256, sizeof(digest_sha256));
514 
515  /*
516  * We mention type and hash in the main log; don't clutter up the files
517  * with anything but the exact dump.
518  */
519  tor_asprintf(&debugfile_base,
520  DESC_DUMP_BASE_FILENAME ".%s", digest_sha256_hex);
521  debugfile = get_datadir_fname2(DESC_DUMP_DATADIR_SUBDIR, debugfile_base);
522 
523  /*
524  * Check if the sandbox is active or will become active; see comment
525  * below at the log message for why.
526  */
527  if (!(sandbox_is_active() || get_options()->Sandbox)) {
528  if (len <= get_options()->MaxUnparseableDescSizeToLog) {
529  if (!dump_desc_fifo_bump_hash(digest_sha256)) {
530  /* Create the directory if needed */
531  dump_desc_create_dir();
532  /* Make sure we've got it */
533  if (have_dump_desc_dir && !problem_with_dump_desc_dir) {
534  /* Write it, and tell the main log about it */
535  write_str_to_file(debugfile, desc, 1);
536  log_info(LD_DIR,
537  "Unable to parse descriptor of type %s with hash %s and "
538  "length %lu. See file %s in data directory for details.",
539  type, digest_sha256_hex, (unsigned long)len,
540  debugfile_base);
541  dump_desc_fifo_add_and_clean(debugfile, digest_sha256, len);
542  /* Since we handed ownership over, don't free debugfile later */
543  debugfile = NULL;
544  } else {
545  /* Problem with the subdirectory */
546  log_info(LD_DIR,
547  "Unable to parse descriptor of type %s with hash %s and "
548  "length %lu. Descriptor not dumped because we had a "
549  "problem creating the " DESC_DUMP_DATADIR_SUBDIR
550  " subdirectory",
551  type, digest_sha256_hex, (unsigned long)len);
552  /* We do have to free debugfile in this case */
553  }
554  } else {
555  /* We already had one with this hash dumped */
556  log_info(LD_DIR,
557  "Unable to parse descriptor of type %s with hash %s and "
558  "length %lu. Descriptor not dumped because one with that "
559  "hash has already been dumped.",
560  type, digest_sha256_hex, (unsigned long)len);
561  /* We do have to free debugfile in this case */
562  }
563  } else {
564  /* Just log that it happened without dumping */
565  log_info(LD_DIR,
566  "Unable to parse descriptor of type %s with hash %s and "
567  "length %lu. Descriptor not dumped because it exceeds maximum"
568  " log size all by itself.",
569  type, digest_sha256_hex, (unsigned long)len);
570  /* We do have to free debugfile in this case */
571  }
572  } else {
573  /*
574  * Not logging because the sandbox is active and seccomp2 apparently
575  * doesn't have a sensible way to allow filenames according to a pattern
576  * match. (If we ever figure out how to say "allow writes to /regex/",
577  * remove this checK).
578  */
579  log_info(LD_DIR,
580  "Unable to parse descriptor of type %s with hash %s and "
581  "length %lu. Descriptor not dumped because the sandbox is "
582  "configured",
583  type, digest_sha256_hex, (unsigned long)len);
584  }
585 
586  tor_free(debugfile_base);
587  tor_free(debugfile);
588 
589  err:
590  return;
591 }
#define SMARTLIST_DEL_CURRENT_KEEPORDER(sl, var)
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
int sandbox_is_active(void)
Definition: sandbox.c:1802
void smartlist_add(smartlist_t *sl, void *element)
Header file for config.c.
#define HEX_DIGEST256_LEN
Definition: crypto_digest.h:37
int strcmpstart(const char *s1, const char *s2)
Definition: util_string.c:206
#define tor_free(p)
Definition: malloc.h:52
void dump_desc_init(void)
Definition: unparseable.c:33
void smartlist_del_keeporder(smartlist_t *sl, int idx)
#define DIGEST256_LEN
Definition: digest_sizes.h:23
Header file for unparseable.c.
tor_assert(buffer)
int tor_memeq(const void *a, const void *b, size_t sz)
Definition: di_ops.c:107
int tor_asprintf(char **strp, const char *fmt,...)
Definition: printf.c:75
Header file for sandbox.c.
Master header file for Tor-specific functionality.
void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:478
#define RFTS_IGNORE_MISSING
Definition: files.h:97
#define LD_DIR
Definition: log.h:86
#define RFTS_BIN
Definition: files.h:95
#define SMARTLIST_FOREACH(sl, type, var, cmd)
int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:506
int crypto_digest256(char *digest, const char *m, size_t len, digest_algorithm_t algorithm)
void smartlist_sort(smartlist_t *sl, int(*compare)(const void **a, const void **b))
Definition: smartlist.c:334
file_status_t file_status(const char *filename)
Definition: files.c:212
void dump_desc_fifo_cleanup(void)
Definition: unparseable.c:234