Tor  0.4.3.0-alpha-dev
dirserv.c
Go to the documentation of this file.
1 /* Copyright (c) 2001-2004, Roger Dingledine.
2  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
3  * Copyright (c) 2007-2019, The Tor Project, Inc. */
4 /* See LICENSE for licensing information */
5 
6 #define DIRSERV_PRIVATE
7 #include "core/or/or.h"
8 
9 #include "app/config/config.h"
17 #include "feature/relay/router.h"
20 
27 
28 #include "lib/compress/compress.h"
29 
30 /**
31  * \file dirserv.c
32  * \brief Directory server core implementation. Manages directory
33  * contents and generates directory documents.
34  *
35  * This module implements most of directory cache functionality, and some of
36  * the directory authority functionality. The directory.c module delegates
37  * here in order to handle incoming requests from clients, via
38  * connection_dirserv_flushed_some() and its kin. In order to save RAM, this
39  * module is responsible for spooling directory objects (in whole or in part)
40  * onto buf_t instances, and then closing the dir_connection_t once the
41  * objects are totally flushed.
42  *
43  * The directory.c module also delegates here for handling descriptor uploads
44  * via dirserv_add_multiple_descriptors().
45  *
46  * Additionally, this module handles some aspects of voting, including:
47  * deciding how to vote on individual flags (based on decisions reached in
48  * rephist.c), of formatting routerstatus lines, and deciding what relays to
49  * include in an authority's vote. (TODO: Those functions could profitably be
50  * split off. They only live in this file because historically they were
51  * shared among the v1, v2, and v3 directory code.)
52  */
53 
54 static void clear_cached_dir(cached_dir_t *d);
56  const uint8_t *fp,
57  int extrainfo);
58 
59 static int spooled_resource_lookup_body(const spooled_resource_t *spooled,
60  int conn_is_encrypted,
61  const uint8_t **body_out,
62  size_t *size_out,
63  time_t *published_out);
65  const spooled_resource_t *spooled,
66  time_t *published_out);
67 static cached_dir_t *lookup_cached_dir_by_fp(const uint8_t *fp);
68 
69 /********************************************************************/
70 
71 /* A set of functions to answer questions about how we'd like to behave
72  * as a directory mirror/client. */
73 
74 /** Return 1 if we fetch our directory material directly from the
75  * authorities, rather than from a mirror. */
76 int
78 {
79  const routerinfo_t *me;
80  uint32_t addr;
81  int refuseunknown;
82  if (options->FetchDirInfoEarly)
83  return 1;
84  if (options->BridgeRelay == 1)
85  return 0;
86  if (server_mode(options) &&
87  router_pick_published_address(options, &addr, 1) < 0)
88  return 1; /* we don't know our IP address; ask an authority. */
89  refuseunknown = ! router_my_exit_policy_is_reject_star() &&
91  if (!dir_server_mode(options) && !refuseunknown)
92  return 0;
93  if (!server_mode(options) || !advertised_server_mode())
94  return 0;
96  if (!me || (!me->supports_tunnelled_dir_requests && !refuseunknown))
97  return 0; /* if we don't service directory requests, return 0 too */
98  return 1;
99 }
100 
101 /** Return 1 if we should fetch new networkstatuses, descriptors, etc
102  * on the "mirror" schedule rather than the "client" schedule.
103  */
104 int
106 {
107  return directory_fetches_from_authorities(options);
108 }
109 
110 /** Return 1 if we should fetch new networkstatuses, descriptors, etc
111  * on a very passive schedule -- waiting long enough for ordinary clients
112  * to probably have the info we want. These would include bridge users,
113  * and maybe others in the future e.g. if a Tor client uses another Tor
114  * client as a directory guard.
115  */
116 int
118 {
119  return options->UseBridges != 0;
120 }
121 
122 /** Return true iff we want to serve certificates for authorities
123  * that we don't acknowledge as authorities ourself.
124  * Use we_want_to_fetch_unknown_auth_certs to check if we want to fetch
125  * and keep these certificates.
126  */
127 int
129 {
130  return dir_server_mode(options) || options->BridgeRelay;
131 }
132 
133 /** Return 1 if we want to fetch and serve descriptors, networkstatuses, etc
134  * Else return 0.
135  * Check options->DirPort_set and directory_permits_begindir_requests()
136  * to see if we are willing to serve these directory documents to others via
137  * the DirPort and begindir-over-ORPort, respectively.
138  *
139  * To check if we should fetch documents, use we_want_to_fetch_flavor and
140  * we_want_to_fetch_unknown_auth_certs instead of this function.
141  */
142 int
144 {
145  if (options->BridgeRelay || dir_server_mode(options))
146  return 1;
147  if (!server_mode(options) || !advertised_server_mode())
148  return 0;
149  /* We need an up-to-date view of network info if we're going to try to
150  * block exit attempts from unknown relays. */
153 }
154 
155 /** Return 1 if we want to allow remote clients to ask us directory
156  * requests via the "begin_dir" interface, which doesn't require
157  * having any separate port open. */
158 int
160 {
161  return options->BridgeRelay != 0 || dir_server_mode(options);
162 }
163 
164 /** Return 1 if we have no need to fetch new descriptors. This generally
165  * happens when we're not a dir cache and we haven't built any circuits
166  * lately.
167  */
168 int
170  time_t now)
171 {
172  return !directory_caches_dir_info(options) &&
173  !options->FetchUselessDescriptors &&
175 }
176 
177 /********************************************************************/
178 
179 /** Map from flavor name to the cached_dir_t for the v3 consensuses that we're
180  * currently serving. */
181 static strmap_t *cached_consensuses = NULL;
182 
183 /** Decrement the reference count on <b>d</b>, and free it if it no longer has
184  * any references. */
185 void
187 {
188  if (!d || --d->refcnt > 0)
189  return;
190  clear_cached_dir(d);
191  tor_free(d);
192 }
193 
194 /** Allocate and return a new cached_dir_t containing the string <b>s</b>,
195  * published at <b>published</b>. */
196 cached_dir_t *
197 new_cached_dir(char *s, time_t published)
198 {
199  cached_dir_t *d = tor_malloc_zero(sizeof(cached_dir_t));
200  d->refcnt = 1;
201  d->dir = s;
202  d->dir_len = strlen(s);
203  d->published = published;
205  d->dir, d->dir_len, ZLIB_METHOD)) {
206  log_warn(LD_BUG, "Error compressing directory");
207  }
208  return d;
209 }
210 
211 /** Remove all storage held in <b>d</b>, but do not free <b>d</b> itself. */
212 static void
214 {
215  tor_free(d->dir);
217  memset(d, 0, sizeof(cached_dir_t));
218 }
219 
220 /** Free all storage held by the cached_dir_t in <b>d</b>. */
221 static void
223 {
224  cached_dir_t *d;
225  if (!_d)
226  return;
227 
228  d = (cached_dir_t *)_d;
230 }
231 
232 /** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that
233  * we're serving with <b>networkstatus</b>, published at <b>published</b>. No
234  * validation is performed. */
235 void
237  size_t networkstatus_len,
238  const char *flavor_name,
239  const common_digests_t *digests,
240  const uint8_t *sha3_as_signed,
241  time_t published)
242 {
243  cached_dir_t *new_networkstatus;
244  cached_dir_t *old_networkstatus;
245  if (!cached_consensuses)
246  cached_consensuses = strmap_new();
247 
248  new_networkstatus =
249  new_cached_dir(tor_memdup_nulterm(networkstatus, networkstatus_len),
250  published);
251  memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t));
252  memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed,
253  DIGEST256_LEN);
254  old_networkstatus = strmap_set(cached_consensuses, flavor_name,
255  new_networkstatus);
256  if (old_networkstatus)
257  cached_dir_decref(old_networkstatus);
258 }
259 
260 /** Return the latest downloaded consensus networkstatus in encoded, signed,
261  * optionally compressed format, suitable for sending to clients. */
263 dirserv_get_consensus,(const char *flavor_name))
264 {
265  if (!cached_consensuses)
266  return NULL;
267  return strmap_get(cached_consensuses, flavor_name);
268 }
269 
270 /** As dirserv_get_routerdescs(), but instead of getting signed_descriptor_t
271  * pointers, adds copies of digests to fps_out, and doesn't use the
272  * /tor/server/ prefix. For a /d/ request, adds descriptor digests; for other
273  * requests, adds identity digests.
274  */
275 int
277  const char *key,
278  dir_spool_source_t source,
279  int conn_is_encrypted,
280  const char **msg_out)
281 {
282  *msg_out = NULL;
283 
284  if (!strcmp(key, "all")) {
285  const routerlist_t *rl = router_get_routerlist();
286  SMARTLIST_FOREACH_BEGIN(rl->routers, const routerinfo_t *, r) {
287  spooled_resource_t *spooled;
288  spooled = spooled_resource_new(source,
289  (const uint8_t *)r->cache_info.identity_digest,
290  DIGEST_LEN);
291  /* Treat "all" requests as if they were unencrypted */
292  conn_is_encrypted = 0;
293  smartlist_add(spool_out, spooled);
294  } SMARTLIST_FOREACH_END(r);
295  } else if (!strcmp(key, "authority")) {
297  if (ri)
298  smartlist_add(spool_out,
299  spooled_resource_new(source,
300  (const uint8_t *)ri->cache_info.identity_digest,
301  DIGEST_LEN));
302  } else if (!strcmpstart(key, "d/")) {
303  key += strlen("d/");
304  dir_split_resource_into_spoolable(key, source, spool_out, NULL,
305  DSR_HEX|DSR_SORT_UNIQ);
306  } else if (!strcmpstart(key, "fp/")) {
307  key += strlen("fp/");
308  dir_split_resource_into_spoolable(key, source, spool_out, NULL,
309  DSR_HEX|DSR_SORT_UNIQ);
310  } else {
311  *msg_out = "Not found";
312  return -1;
313  }
314 
315  if (! conn_is_encrypted) {
316  /* Remove anything that insists it not be sent unencrypted. */
317  SMARTLIST_FOREACH_BEGIN(spool_out, spooled_resource_t *, spooled) {
318  const uint8_t *body = NULL;
319  size_t bodylen = 0;
320  int r = spooled_resource_lookup_body(spooled, conn_is_encrypted,
321  &body, &bodylen, NULL);
322  if (r < 0 || body == NULL || bodylen == 0) {
323  SMARTLIST_DEL_CURRENT(spool_out, spooled);
324  spooled_resource_free(spooled);
325  }
326  } SMARTLIST_FOREACH_END(spooled);
327  }
328 
329  if (!smartlist_len(spool_out)) {
330  *msg_out = "Servers unavailable";
331  return -1;
332  }
333  return 0;
334 }
335 
336 /** Add a signed_descriptor_t to <b>descs_out</b> for each router matching
337  * <b>key</b>. The key should be either
338  * - "/tor/server/authority" for our own routerinfo;
339  * - "/tor/server/all" for all the routerinfos we have, concatenated;
340  * - "/tor/server/fp/FP" where FP is a plus-separated sequence of
341  * hex identity digests; or
342  * - "/tor/server/d/D" where D is a plus-separated sequence
343  * of server descriptor digests, in hex.
344  *
345  * Return 0 if we found some matching descriptors, or -1 if we do not
346  * have any descriptors, no matching descriptors, or if we did not
347  * recognize the key (URL).
348  * If -1 is returned *<b>msg</b> will be set to an appropriate error
349  * message.
350  *
351  * XXXX rename this function. It's only called from the controller.
352  * XXXX in fact, refactor this function, merging as much as possible.
353  */
354 int
355 dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
356  const char **msg)
357 {
358  *msg = NULL;
359 
360  if (!strcmp(key, "/tor/server/all")) {
363  smartlist_add(descs_out, &(r->cache_info)));
364  } else if (!strcmp(key, "/tor/server/authority")) {
366  if (ri)
367  smartlist_add(descs_out, (void*) &(ri->cache_info));
368  } else if (!strcmpstart(key, "/tor/server/d/")) {
369  smartlist_t *digests = smartlist_new();
370  key += strlen("/tor/server/d/");
371  dir_split_resource_into_fingerprints(key, digests, NULL,
372  DSR_HEX|DSR_SORT_UNIQ);
373  SMARTLIST_FOREACH(digests, const char *, d,
374  {
376  if (sd)
377  smartlist_add(descs_out,sd);
378  });
379  SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
380  smartlist_free(digests);
381  } else if (!strcmpstart(key, "/tor/server/fp/")) {
382  smartlist_t *digests = smartlist_new();
383  time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
384  key += strlen("/tor/server/fp/");
385  dir_split_resource_into_fingerprints(key, digests, NULL,
386  DSR_HEX|DSR_SORT_UNIQ);
387  SMARTLIST_FOREACH_BEGIN(digests, const char *, d) {
388  if (router_digest_is_me(d)) {
389  /* calling router_get_my_routerinfo() to make sure it exists */
391  if (ri)
392  smartlist_add(descs_out, (void*) &(ri->cache_info));
393  } else {
394  const routerinfo_t *ri = router_get_by_id_digest(d);
395  /* Don't actually serve a descriptor that everyone will think is
396  * expired. This is an (ugly) workaround to keep buggy 0.1.1.10
397  * Tors from downloading descriptors that they will throw away.
398  */
399  if (ri && ri->cache_info.published_on > cutoff)
400  smartlist_add(descs_out, (void*) &(ri->cache_info));
401  }
402  } SMARTLIST_FOREACH_END(d);
403  SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
404  smartlist_free(digests);
405  } else {
406  *msg = "Key not recognized";
407  return -1;
408  }
409 
410  if (!smartlist_len(descs_out)) {
411  *msg = "Servers unavailable";
412  return -1;
413  }
414  return 0;
415 }
416 
417 /* ==========
418  * Spooling code.
419  * ========== */
420 
422 spooled_resource_new(dir_spool_source_t source,
423  const uint8_t *digest, size_t digestlen)
424 {
425  spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t));
426  spooled->spool_source = source;
427  switch (source) {
428  case DIR_SPOOL_NETWORKSTATUS:
429  spooled->spool_eagerly = 0;
430  break;
431  case DIR_SPOOL_SERVER_BY_DIGEST:
432  case DIR_SPOOL_SERVER_BY_FP:
433  case DIR_SPOOL_EXTRA_BY_DIGEST:
434  case DIR_SPOOL_EXTRA_BY_FP:
435  case DIR_SPOOL_MICRODESC:
436  default:
437  spooled->spool_eagerly = 1;
438  break;
439  case DIR_SPOOL_CONSENSUS_CACHE_ENTRY:
440  tor_assert_unreached();
441  break;
442  }
443  tor_assert(digestlen <= sizeof(spooled->digest));
444  if (digest)
445  memcpy(spooled->digest, digest, digestlen);
446  return spooled;
447 }
448 
449 /**
450  * Create a new spooled_resource_t to spool the contents of <b>entry</b> to
451  * the user. Return the spooled object on success, or NULL on failure (which
452  * is probably caused by a failure to map the body of the item from disk).
453  *
454  * Adds a reference to entry's reference counter.
455  */
458 {
459  spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t));
460  spooled->spool_source = DIR_SPOOL_CONSENSUS_CACHE_ENTRY;
461  spooled->spool_eagerly = 0;
463  spooled->consensus_cache_entry = entry;
464 
465  int r = consensus_cache_entry_get_body(entry,
466  &spooled->cce_body,
467  &spooled->cce_len);
468  if (r == 0) {
469  return spooled;
470  } else {
471  spooled_resource_free(spooled);
472  return NULL;
473  }
474 }
475 
476 /** Release all storage held by <b>spooled</b>. */
477 void
479 {
480  if (spooled == NULL)
481  return;
482 
483  if (spooled->cached_dir_ref) {
485  }
486 
487  if (spooled->consensus_cache_entry) {
489  }
490 
491  tor_free(spooled);
492 }
493 
494 /** When spooling data from a cached_dir_t object, we always add
495  * at least this much. */
496 #define DIRSERV_CACHED_DIR_CHUNK_SIZE 8192
497 
498 /** Return an compression ratio for compressing objects from <b>source</b>.
499  */
500 static double
502 {
503  /* We should put in better estimates here, depending on the number of
504  objects and their type */
505  (void) source;
506  return 0.5;
507 }
508 
509 /** Return an estimated number of bytes needed for transmitting the
510  * resource in <b>spooled</b> on <b>conn</b>
511  *
512  * As a convenient side-effect, set *<b>published_out</b> to the resource's
513  * publication time.
514  */
515 static size_t
517  dir_connection_t *conn,
518  int compressed,
519  time_t *published_out)
520 {
521  if (spooled->spool_eagerly) {
522  const uint8_t *body = NULL;
523  size_t bodylen = 0;
524  int r = spooled_resource_lookup_body(spooled,
526  &body, &bodylen,
527  published_out);
528  if (r == -1 || body == NULL || bodylen == 0)
529  return 0;
530  if (compressed) {
531  double ratio = estimate_compression_ratio(spooled->spool_source);
532  bodylen = (size_t)(bodylen * ratio);
533  }
534  return bodylen;
535  } else {
536  cached_dir_t *cached;
537  if (spooled->consensus_cache_entry) {
538  if (published_out) {
540  spooled->consensus_cache_entry, published_out);
541  }
542 
543  return spooled->cce_len;
544  }
545  if (spooled->cached_dir_ref) {
546  cached = spooled->cached_dir_ref;
547  } else {
548  cached = spooled_resource_lookup_cached_dir(spooled,
549  published_out);
550  }
551  if (cached == NULL) {
552  return 0;
553  }
554  size_t result = compressed ? cached->dir_compressed_len : cached->dir_len;
555  return result;
556  }
557 }
558 
559 /** Return code for spooled_resource_flush_some */
560 typedef enum {
561  SRFS_ERR = -1,
562  SRFS_MORE = 0,
563  SRFS_DONE
565 
566 /** Flush some or all of the bytes from <b>spooled</b> onto <b>conn</b>.
567  * Return SRFS_ERR on error, SRFS_MORE if there are more bytes to flush from
568  * this spooled resource, or SRFS_DONE if we are done flushing this spooled
569  * resource.
570  */
573  dir_connection_t *conn)
574 {
575  if (spooled->spool_eagerly) {
576  /* Spool_eagerly resources are sent all-at-once. */
577  const uint8_t *body = NULL;
578  size_t bodylen = 0;
579  int r = spooled_resource_lookup_body(spooled,
581  &body, &bodylen, NULL);
582  if (r == -1 || body == NULL || bodylen == 0) {
583  /* Absent objects count as "done". */
584  return SRFS_DONE;
585  }
586 
587  connection_dir_buf_add((const char*)body, bodylen, conn, 0);
588 
589  return SRFS_DONE;
590  } else {
591  cached_dir_t *cached = spooled->cached_dir_ref;
593  if (cached == NULL && cce == NULL) {
594  /* The cached_dir_t hasn't been materialized yet. So let's look it up. */
595  cached = spooled->cached_dir_ref =
596  spooled_resource_lookup_cached_dir(spooled, NULL);
597  if (!cached) {
598  /* Absent objects count as done. */
599  return SRFS_DONE;
600  }
601  ++cached->refcnt;
602  tor_assert_nonfatal(spooled->cached_dir_offset == 0);
603  }
604 
605  if (BUG(!cached && !cce))
606  return SRFS_DONE;
607 
608  int64_t total_len;
609  const char *ptr;
610  if (cached) {
611  total_len = cached->dir_compressed_len;
612  ptr = cached->dir_compressed;
613  } else {
614  total_len = spooled->cce_len;
615  ptr = (const char *)spooled->cce_body;
616  }
617  /* How many bytes left to flush? */
618  int64_t remaining;
619  remaining = total_len - spooled->cached_dir_offset;
620  if (BUG(remaining < 0))
621  return SRFS_ERR;
622  ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining);
623 
625  bytes, conn, 0);
626 
627  spooled->cached_dir_offset += bytes;
628  if (spooled->cached_dir_offset >= (off_t)total_len) {
629  return SRFS_DONE;
630  } else {
631  return SRFS_MORE;
632  }
633  }
634 }
635 
636 /** Helper: find the cached_dir_t for a spooled_resource_t, for
637  * sending it to <b>conn</b>. Set *<b>published_out</b>, if provided,
638  * to the published time of the cached_dir_t.
639  *
640  * DOES NOT increase the reference count on the result. Callers must do that
641  * themselves if they mean to hang on to it.
642  */
643 static cached_dir_t *
645  time_t *published_out)
646 {
647  tor_assert(spooled->spool_eagerly == 0);
649  if (d != NULL) {
650  if (published_out)
651  *published_out = d->published;
652  }
653  return d;
654 }
655 
656 /** Helper: Look up the body for an eagerly-served spooled_resource. If
657  * <b>conn_is_encrypted</b> is false, don't look up any resource that
658  * shouldn't be sent over an unencrypted connection. On success, set
659  * <b>body_out</b>, <b>size_out</b>, and <b>published_out</b> to refer
660  * to the resource's body, size, and publication date, and return 0.
661  * On failure return -1. */
662 static int
664  int conn_is_encrypted,
665  const uint8_t **body_out,
666  size_t *size_out,
667  time_t *published_out)
668 {
669  tor_assert(spooled->spool_eagerly == 1);
670 
671  const signed_descriptor_t *sd = NULL;
672 
673  switch (spooled->spool_source) {
674  case DIR_SPOOL_EXTRA_BY_FP: {
675  sd = get_signed_descriptor_by_fp(spooled->digest, 1);
676  break;
677  }
678  case DIR_SPOOL_SERVER_BY_FP: {
679  sd = get_signed_descriptor_by_fp(spooled->digest, 0);
680  break;
681  }
682  case DIR_SPOOL_SERVER_BY_DIGEST: {
683  sd = router_get_by_descriptor_digest((const char *)spooled->digest);
684  break;
685  }
686  case DIR_SPOOL_EXTRA_BY_DIGEST: {
687  sd = extrainfo_get_by_descriptor_digest((const char *)spooled->digest);
688  break;
689  }
690  case DIR_SPOOL_MICRODESC: {
693  (const char *)spooled->digest);
694  if (! md || ! md->body) {
695  return -1;
696  }
697  *body_out = (const uint8_t *)md->body;
698  *size_out = md->bodylen;
699  if (published_out)
700  *published_out = TIME_MAX;
701  return 0;
702  }
703  case DIR_SPOOL_NETWORKSTATUS:
704  case DIR_SPOOL_CONSENSUS_CACHE_ENTRY:
705  default:
706  /* LCOV_EXCL_START */
707  tor_assert_nonfatal_unreached();
708  return -1;
709  /* LCOV_EXCL_STOP */
710  }
711 
712  /* If we get here, then we tried to set "sd" to a signed_descriptor_t. */
713 
714  if (sd == NULL) {
715  return -1;
716  }
717  if (sd->send_unencrypted == 0 && ! conn_is_encrypted) {
718  /* we did this check once before (so we could have an accurate size
719  * estimate and maybe send a 404 if somebody asked for only bridges on
720  * a connection), but we need to do it again in case a previously
721  * unknown bridge descriptor has shown up between then and now. */
722  return -1;
723  }
724  *body_out = (const uint8_t *) signed_descriptor_get_body(sd);
725  *size_out = sd->signed_descriptor_len;
726  if (published_out)
727  *published_out = sd->published_on;
728  return 0;
729 }
730 
731 /** Given a fingerprint <b>fp</b> which is either set if we're looking for a
732  * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded
733  * flavor name if we want a flavored v3 status, return a pointer to the
734  * appropriate cached dir object, or NULL if there isn't one available. */
735 static cached_dir_t *
736 lookup_cached_dir_by_fp(const uint8_t *fp)
737 {
738  cached_dir_t *d = NULL;
739  if (tor_digest_is_zero((const char *)fp) && cached_consensuses) {
740  d = strmap_get(cached_consensuses, "ns");
741  } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses) {
742  /* this here interface is a nasty hack: we're shoving a flavor into
743  * a digest field. */
744  d = strmap_get(cached_consensuses, (const char *)fp);
745  }
746  return d;
747 }
748 
749 /** Try to guess the number of bytes that will be needed to send the
750  * spooled objects for <b>conn</b>'s outgoing spool. In the process,
751  * remove every element of the spool that refers to an absent object, or
752  * which was published earlier than <b>cutoff</b>. Set *<b>size_out</b>
753  * to the number of bytes, and *<b>n_expired_out</b> to the number of
754  * objects removed for being too old. */
755 void
757  time_t cutoff,
758  int compression,
759  size_t *size_out,
760  int *n_expired_out)
761 {
762  if (BUG(!conn))
763  return;
764 
765  smartlist_t *spool = conn->spool;
766  if (!spool) {
767  if (size_out)
768  *size_out = 0;
769  if (n_expired_out)
770  *n_expired_out = 0;
771  return;
772  }
773  int n_expired = 0;
774  uint64_t total = 0;
775  SMARTLIST_FOREACH_BEGIN(spool, spooled_resource_t *, spooled) {
776  time_t published = TIME_MAX;
777  size_t sz = spooled_resource_estimate_size(spooled, conn,
778  compression, &published);
779  if (published < cutoff) {
780  ++n_expired;
781  SMARTLIST_DEL_CURRENT(spool, spooled);
782  spooled_resource_free(spooled);
783  } else if (sz == 0) {
784  SMARTLIST_DEL_CURRENT(spool, spooled);
785  spooled_resource_free(spooled);
786  } else {
787  total += sz;
788  }
789  } SMARTLIST_FOREACH_END(spooled);
790 
791  if (size_out) {
792  *size_out = (total > SIZE_MAX) ? SIZE_MAX : (size_t)total;
793  }
794  if (n_expired_out)
795  *n_expired_out = n_expired;
796 }
797 
798 /** Helper: used to sort a connection's spool. */
799 static int
800 dirserv_spool_sort_comparison_(const void **a_, const void **b_)
801 {
802  const spooled_resource_t *a = *a_;
803  const spooled_resource_t *b = *b_;
804  return fast_memcmp(a->digest, b->digest, sizeof(a->digest));
805 }
806 
807 /** Sort all the entries in <b>conn</b> by digest. */
808 void
810 {
811  if (conn->spool == NULL)
812  return;
814 }
815 
816 /** Return the cache-info for identity fingerprint <b>fp</b>, or
817  * its extra-info document if <b>extrainfo</b> is true. Return
818  * NULL if not found or if the descriptor is older than
819  * <b>publish_cutoff</b>. */
820 static const signed_descriptor_t *
821 get_signed_descriptor_by_fp(const uint8_t *fp, int extrainfo)
822 {
823  if (router_digest_is_me((const char *)fp)) {
824  if (extrainfo)
825  return &(router_get_my_extrainfo()->cache_info);
826  else
827  return &(router_get_my_routerinfo()->cache_info);
828  } else {
829  const routerinfo_t *ri = router_get_by_id_digest((const char *)fp);
830  if (ri) {
831  if (extrainfo)
833  ri->cache_info.extra_info_digest);
834  else
835  return &ri->cache_info;
836  }
837  }
838  return NULL;
839 }
840 
841 /** When we're spooling data onto our outbuf, add more whenever we dip
842  * below this threshold. */
843 #define DIRSERV_BUFFER_MIN 16384
844 
845 /**
846  * Called whenever we have flushed some directory data in state
847  * SERVER_WRITING, or whenever we want to fill the buffer with initial
848  * directory data (so that subsequent writes will occur, and trigger this
849  * function again.)
850  *
851  * Return 0 on success, and -1 on failure.
852  */
853 int
855 {
857  if (conn->spool == NULL)
858  return 0;
859 
860  while (connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN &&
861  smartlist_len(conn->spool)) {
862  spooled_resource_t *spooled =
863  smartlist_get(conn->spool, smartlist_len(conn->spool)-1);
865  status = spooled_resource_flush_some(spooled, conn);
866  if (status == SRFS_ERR) {
867  return -1;
868  } else if (status == SRFS_MORE) {
869  return 0;
870  }
871  tor_assert(status == SRFS_DONE);
872 
873  /* If we're here, we're done flushing this resource. */
874  tor_assert(smartlist_pop_last(conn->spool) == spooled);
875  spooled_resource_free(spooled);
876  }
877 
878  if (smartlist_len(conn->spool) > 0) {
879  /* We're still spooling something. */
880  return 0;
881  }
882 
883  /* If we get here, we're done. */
884  smartlist_free(conn->spool);
885  conn->spool = NULL;
886  if (conn->compress_state) {
887  /* Flush the compression state: there could be more bytes pending in there,
888  * and we don't want to omit bytes. */
889  connection_buf_add_compress("", 0, conn, 1);
890  tor_compress_free(conn->compress_state);
891  conn->compress_state = NULL;
892  }
893  return 0;
894 }
895 
896 /** Remove every element from <b>conn</b>'s outgoing spool, and delete
897  * the spool. */
898 void
900 {
901  if (!conn || ! conn->spool)
902  return;
904  spooled_resource_free(s));
905  smartlist_free(conn->spool);
906  conn->spool = NULL;
907 }
908 
909 /** Release all storage used by the directory server. */
910 void
912 {
913  strmap_free(cached_consensuses, free_cached_dir_);
914  cached_consensuses = NULL;
915 }
dir_spool_source_bitfield_t spool_source
Definition: dirserv.h:51
Header file for dirserv.c.
struct tor_compress_state_t * compress_state
int directory_fetches_dir_info_early(const or_options_t *options)
Definition: dirserv.c:105
A relay's extra-info structure.
extrainfo_t * router_get_my_extrainfo(void)
Definition: router.c:1686
char * body
Definition: microdesc_st.h:56
Router descriptor structure.
void dir_conn_clear_spool(dir_connection_t *conn)
Definition: dirserv.c:899
void spooled_resource_free_(spooled_resource_t *spooled)
Definition: dirserv.c:478
Header for conscache.c.
smartlist_t * routers
Definition: routerlist_st.h:32
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
#define TO_CONN(c)
Definition: or.h:735
#define MOCK_IMPL(rv, funcname, arglist)
Definition: testsupport.h:133
size_t bodylen
Definition: microdesc_st.h:58
Header file for connection.c.
static cached_dir_t * spooled_resource_lookup_cached_dir(const spooled_resource_t *spooled, time_t *published_out)
Definition: dirserv.c:644
int should_refuse_unknown_exits(const or_options_t *options)
Definition: router.c:1325
Headers for compress.c.
signed_descriptor_t * extrainfo_get_by_descriptor_digest(const char *digest)
Definition: routerlist.c:724
int consensus_cache_entry_get_body(const consensus_cache_entry_t *ent, const uint8_t **body_out, size_t *sz_out)
Definition: conscache.c:422
Header file for directory.c.
Microdescriptor structure.
void smartlist_add(smartlist_t *sl, void *element)
void * smartlist_pop_last(smartlist_t *sl)
const char * signed_descriptor_get_body(const signed_descriptor_t *desc)
Definition: routerlist.c:794
int dirserv_get_routerdesc_spool(smartlist_t *spool_out, const char *key, dir_spool_source_t source, int conn_is_encrypted, const char **msg_out)
Definition: dirserv.c:276
int directory_caches_unknown_auth_certs(const or_options_t *options)
Definition: dirserv.c:128
time_t published
Definition: cached_dir_st.h:22
char * dir_compressed
Definition: cached_dir_st.h:19
Header file for config.c.
char identity_digest[DIGEST_LEN]
#define tor_assert(expr)
Definition: util_bug.h:102
int router_pick_published_address(const or_options_t *options, uint32_t *addr, int cache_only)
Definition: router.c:1715
Header file for microdesc.c.
int directory_fetches_from_authorities(const or_options_t *options)
Definition: dirserv.c:77
common_digests_t digests
Definition: cached_dir_st.h:23
int strcmpstart(const char *s1, const char *s2)
Definition: util_string.c:206
size_t dir_compressed_len
Definition: cached_dir_st.h:21
int directory_too_idle_to_fetch_descriptors(const or_options_t *options, time_t now)
Definition: dirserv.c:169
#define tor_free(p)
Definition: malloc.h:52
Definition: conscache.c:32
static int dirserv_spool_sort_comparison_(const void **a_, const void **b_)
Definition: dirserv.c:800
#define SMARTLIST_DEL_CURRENT(sl, var)
Cached large directory object structure.
int connection_dirserv_flushed_some(dir_connection_t *conn)
Definition: dirserv.c:854
static void clear_cached_dir(cached_dir_t *d)
Definition: dirserv.c:213
smartlist_t * smartlist_new(void)
unsigned spool_eagerly
Definition: dirserv.h:47
int dir_server_mode(const or_options_t *options)
Definition: routermode.c:23
spooled_resource_t * spooled_resource_new_from_cache_entry(consensus_cache_entry_t *entry)
Definition: dirserv.c:457
static strmap_t * cached_consensuses
Definition: dirserv.c:181
#define DIGEST256_LEN
Definition: digest_sizes.h:23
int directory_permits_begindir_requests(const or_options_t *options)
Definition: dirserv.c:159
static const signed_descriptor_t * get_signed_descriptor_by_fp(const uint8_t *fp, int extrainfo)
Definition: dirserv.c:821
int directory_fetches_dir_info_later(const or_options_t *options)
Definition: dirserv.c:117
off_t cached_dir_offset
Definition: dirserv.h:71
size_t dir_len
Definition: cached_dir_st.h:20
uint8_t digest[DIGEST256_LEN]
Definition: dirserv.h:55
void cached_dir_decref(cached_dir_t *d)
Definition: dirserv.c:186
int tor_compress(char **out, size_t *out_len, const char *in, size_t in_len, compress_method_t method)
Definition: compress.c:242
Header file for routermode.c.
const routerinfo_t * router_get_by_id_digest(const char *digest)
Definition: routerlist.c:690
static double estimate_compression_ratio(dir_spool_source_t source)
Definition: dirserv.c:501
#define DIGEST_LEN
Definition: digest_sizes.h:20
dir_spool_source_t
Definition: dirserv.h:20
spooled_resource_flush_status_t
Definition: dirserv.c:560
Master header file for Tor-specific functionality.
int rep_hist_circbuilding_dormant(time_t now)
void dirserv_free_all(void)
Definition: dirserv.c:911
static void free_cached_dir_(void *_d)
Definition: dirserv.c:222
void consensus_cache_entry_incref(consensus_cache_entry_t *ent)
Definition: conscache.c:340
static size_t spooled_resource_estimate_size(const spooled_resource_t *spooled, dir_connection_t *conn, int compressed, time_t *published_out)
Definition: dirserv.c:516
struct cached_dir_t * cached_dir_ref
Definition: dirserv.h:60
void dirserv_spool_sort(dir_connection_t *conn)
Definition: dirserv.c:809
signed_descriptor_t * router_get_by_descriptor_digest(const char *digest)
Definition: routerlist.c:698
#define DIRSERV_CACHED_DIR_CHUNK_SIZE
Definition: dirserv.c:496
int tor_digest_is_zero(const char *digest)
Definition: util_string.c:96
Client/server directory connection structure.
int consensus_cache_entry_get_valid_after(const consensus_cache_entry_t *ent, time_t *out)
Definition: consdiffmgr.c:1967
microdesc_t * microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
Definition: microdesc.c:944
void connection_dir_buf_add(const char *string, size_t len, dir_connection_t *dir_conn, int done)
Definition: connection.c:4372
cached_dir_t * new_cached_dir(char *s, time_t published)
Definition: dirserv.c:197
cached_dir_t * dirserv_get_consensus(const char *flavor_name)
Definition: dirserv.c:263
#define ROUTER_MAX_AGE_TO_PUBLISH
Definition: or.h:162
const routerinfo_t * router_get_my_routerinfo(void)
Definition: router.c:1623
microdesc_cache_t * get_microdesc_cache(void)
Definition: microdesc.c:249
#define SMARTLIST_FOREACH(sl, type, var, cmd)
int connection_dir_is_encrypted(const dir_connection_t *conn)
Definition: directory.c:168
Header file for router.c.
int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, const char **msg)
Definition: dirserv.c:355
void dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn, time_t cutoff, int compression, size_t *size_out, int *n_expired_out)
Definition: dirserv.c:756
int dir_split_resource_into_fingerprints(const char *resource, smartlist_t *fp_out, int *compressed_out, int flags)
Definition: directory.c:628
int router_digest_is_me(const char *digest)
Definition: router.c:1586
uint8_t digest_sha3_as_signed[DIGEST256_LEN]
Definition: cached_dir_st.h:25
int router_my_exit_policy_is_reject_star(void)
Definition: router.c:1574
#define fast_memcmp(a, b, c)
Definition: di_ops.h:28
struct consensus_cache_entry_t * consensus_cache_entry
Definition: dirserv.h:65
static int spooled_resource_lookup_body(const spooled_resource_t *spooled, int conn_is_encrypted, const uint8_t **body_out, size_t *size_out, time_t *published_out)
Definition: dirserv.c:663
int dir_split_resource_into_spoolable(const char *resource, dir_spool_source_t source, smartlist_t *spool_out, int *compressed_out, int flags)
Definition: directory.c:710
int directory_caches_dir_info(const or_options_t *options)
Definition: dirserv.c:143
char extra_info_digest[DIGEST_LEN]
#define DIRSERV_BUFFER_MIN
Definition: dirserv.c:843
static cached_dir_t * lookup_cached_dir_by_fp(const uint8_t *fp)
Definition: dirserv.c:736
void consensus_cache_entry_decref(consensus_cache_entry_t *ent)
Definition: conscache.c:355
static spooled_resource_flush_status_t spooled_resource_flush_some(spooled_resource_t *spooled, dir_connection_t *conn)
Definition: dirserv.c:572
smartlist_t * spool
int server_mode(const or_options_t *options)
Definition: routermode.c:34
void dirserv_set_cached_consensus_networkstatus(const char *networkstatus, size_t networkstatus_len, const char *flavor_name, const common_digests_t *digests, const uint8_t *sha3_as_signed, time_t published)
Definition: dirserv.c:236
Header for consdiffmgr.c.
void smartlist_sort(smartlist_t *sl, int(*compare)(const void **a, const void **b))
Definition: smartlist.c:334
#define DIR_CONN_STATE_SERVER_WRITING
Definition: directory.h:29
routerlist_t * router_get_routerlist(void)
Definition: routerlist.c:809
uint8_t state
Definition: connection_st.h:49
#define LD_BUG
Definition: log.h:86
Router descriptor list structure.
Header file for predict_ports.c.
int advertised_server_mode(void)
Definition: routermode.c:55
Header file for routerlist.c.
unsigned int supports_tunnelled_dir_requests
Definition: routerinfo_st.h:88
int FetchUselessDescriptors