tor  0.4.2.0-alpha-dev
dircache.c
1 /* Copyright (c) 2001-2004, Roger Dingledine.
2  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
3  * Copyright (c) 2007-2019, The Tor Project, Inc. */
4 /* See LICENSE for licensing information */
5 
6 #define DIRCACHE_PRIVATE
7 
8 #include "core/or/or.h"
9 
10 #include "app/config/config.h"
12 #include "core/or/relay.h"
16 #include "feature/dircache/conscache.h"
17 #include "feature/dircache/consdiffmgr.h"
22 #include "feature/hs/hs_cache.h"
27 #include "feature/rend/rendcache.h"
29 #include "feature/stats/rephist.h"
30 #include "lib/compress/compress.h"
31 
32 #include "feature/dircache/cached_dir_st.h"
33 #include "feature/dircommon/dir_connection_st.h"
34 #include "feature/nodelist/authority_cert_st.h"
35 #include "feature/nodelist/networkstatus_st.h"
36 #include "feature/nodelist/routerinfo_st.h"
37 
40 #define MAX_DIR_UL_SIZE ((1<<24)-1) /* 16MB-1 */
41 
44 #define FULL_DIR_CACHE_LIFETIME (60*60)
45 #define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
46 #define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60)
47 #define NETWORKSTATUS_CACHE_LIFETIME (5*60)
48 #define ROUTERDESC_CACHE_LIFETIME (30*60)
49 #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
50 #define ROBOTS_CACHE_LIFETIME (24*60*60)
51 #define MICRODESC_CACHE_LIFETIME (48*60*60)
52 /* Bandwidth files change every hour. */
53 #define BANDWIDTH_CACHE_LIFETIME (30*60)
54 
63 STATIC int
64 parse_http_url(const char *headers, char **url)
65 {
66  char *command = NULL;
67  if (parse_http_command(headers, &command, url) < 0) {
68  return -1;
69  }
70  if (strcmpstart(*url, "/tor/")) {
71  char *new_url = NULL;
72  tor_asprintf(&new_url, "/tor%s%s",
73  *url[0] == '/' ? "" : "/",
74  *url);
75  tor_free(*url);
76  *url = new_url;
77  }
78  tor_free(command);
79  return 0;
80 }
81 
85 static void
86 write_short_http_response(dir_connection_t *conn, int status,
87  const char *reason_phrase)
88 {
89  char *buf = NULL;
90  char *datestring = NULL;
91 
92  IF_BUG_ONCE(!reason_phrase) { /* bullet-proofing */
93  reason_phrase = "unspecified";
94  }
95 
96  if (server_mode(get_options())) {
97  /* include the Date: header, but only if we're a relay or bridge */
98  char datebuf[RFC1123_TIME_LEN+1];
99  format_rfc1123_time(datebuf, time(NULL));
100  tor_asprintf(&datestring, "Date: %s\r\n", datebuf);
101  }
102 
103  tor_asprintf(&buf, "HTTP/1.0 %d %s\r\n%s\r\n",
104  status, reason_phrase, datestring?datestring:"");
105 
106  log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase);
107  connection_buf_add(buf, strlen(buf), TO_CONN(conn));
108 
109  tor_free(datestring);
110  tor_free(buf);
111 }
112 
120 static void
121 write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
122  const char *type, const char *encoding,
123  const char *extra_headers,
124  long cache_lifetime)
125 {
126  char date[RFC1123_TIME_LEN+1];
127  time_t now = approx_time();
128  buf_t *buf = buf_new_with_capacity(1024);
129 
130  tor_assert(conn);
131 
132  format_rfc1123_time(date, now);
133 
134  buf_add_printf(buf, "HTTP/1.0 200 OK\r\nDate: %s\r\n", date);
135  if (type) {
136  buf_add_printf(buf, "Content-Type: %s\r\n", type);
137  }
138  if (!is_local_addr(&conn->base_.addr)) {
139  /* Don't report the source address for a nearby/private connection.
140  * Otherwise we tend to mis-report in cases where incoming ports are
141  * being forwarded to a Tor server running behind the firewall. */
142  buf_add_printf(buf, X_ADDRESS_HEADER "%s\r\n", conn->base_.address);
143  }
144  if (encoding) {
145  buf_add_printf(buf, "Content-Encoding: %s\r\n", encoding);
146  }
147  if (length >= 0) {
148  buf_add_printf(buf, "Content-Length: %ld\r\n", (long)length);
149  }
150  if (cache_lifetime > 0) {
151  char expbuf[RFC1123_TIME_LEN+1];
152  format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime));
153  /* We could say 'Cache-control: max-age=%d' here if we start doing
154  * http/1.1 */
155  buf_add_printf(buf, "Expires: %s\r\n", expbuf);
156  } else if (cache_lifetime == 0) {
157  /* We could say 'Cache-control: no-cache' here if we start doing
158  * http/1.1 */
159  buf_add_string(buf, "Pragma: no-cache\r\n");
160  }
161  if (extra_headers) {
162  buf_add_string(buf, extra_headers);
163  }
164  buf_add_string(buf, "\r\n");
165 
166  connection_buf_add_buf(TO_CONN(conn), buf);
167  buf_free(buf);
168 }
169 
172 static void
173 write_http_response_headers(dir_connection_t *conn, ssize_t length,
174  compress_method_t method,
175  const char *extra_headers, long cache_lifetime)
176 {
177  write_http_response_header_impl(conn, length,
178  "text/plain",
180  extra_headers,
181  cache_lifetime);
182 }
183 
185 static void
186 write_http_response_header(dir_connection_t *conn, ssize_t length,
187  compress_method_t method,
188  long cache_lifetime)
189 {
190  write_http_response_headers(conn, length, method, NULL, cache_lifetime);
191 }
192 
195 static compress_method_t srv_meth_pref_precompressed[] = {
196  LZMA_METHOD,
197  ZSTD_METHOD,
198  ZLIB_METHOD,
199  GZIP_METHOD,
200  NO_METHOD
201 };
202 
205 static compress_method_t srv_meth_pref_streaming_compression[] = {
206  ZSTD_METHOD,
207  ZLIB_METHOD,
208  GZIP_METHOD,
209  NO_METHOD
210 };
211 
215 STATIC unsigned
216 parse_accept_encoding_header(const char *h)
217 {
218  unsigned result = (1u << NO_METHOD);
219  smartlist_t *methods = smartlist_new();
220  smartlist_split_string(methods, h, ",",
221  SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0);
222 
223  SMARTLIST_FOREACH_BEGIN(methods, const char *, m) {
225  if (method != UNKNOWN_METHOD) {
226  tor_assert(((unsigned)method) < 8*sizeof(unsigned));
227  result |= (1u << method);
228  }
229  } SMARTLIST_FOREACH_END(m);
230  SMARTLIST_FOREACH_BEGIN(methods, char *, m) {
231  tor_free(m);
232  } SMARTLIST_FOREACH_END(m);
233  smartlist_free(methods);
234  return result;
235 }
236 
251 static int
252 client_likes_consensus(const struct consensus_cache_entry_t *ent,
253  const char *want_url)
254 {
255  smartlist_t *voters = smartlist_new();
256  int need_at_least;
257  int have = 0;
258 
259  if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) {
260  smartlist_free(voters);
261  return 1; // We don't know the voters; assume the client won't mind. */
262  }
263 
264  smartlist_t *want_authorities = smartlist_new();
265  dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0);
266  need_at_least = smartlist_len(want_authorities)/2+1;
267 
268  SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) {
269 
270  SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) {
271  if (!strcasecmpstart(digest, want_digest)) {
272  have++;
273  break;
274  };
275  } SMARTLIST_FOREACH_END(digest);
276 
277  /* early exit, if we already have enough */
278  if (have >= need_at_least)
279  break;
280  } SMARTLIST_FOREACH_END(want_digest);
281 
282  SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d));
283  smartlist_free(want_authorities);
284  SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp));
285  smartlist_free(voters);
286  return (have >= need_at_least);
287 }
288 
291 STATIC compression_level_t
292 choose_compression_level(ssize_t n_bytes)
293 {
295  return HIGH_COMPRESSION; /* we have plenty of RAM. */
296  } else if (n_bytes < 0) {
297  return HIGH_COMPRESSION; /* unknown; might be big. */
298  } else if (n_bytes < 1024) {
299  return LOW_COMPRESSION;
300  } else if (n_bytes < 2048) {
301  return MEDIUM_COMPRESSION;
302  } else {
303  return HIGH_COMPRESSION;
304  }
305 }
306 
308 typedef struct get_handler_args_t {
316  const char *url;
318  const char *headers;
320 
331 typedef struct url_table_ent_s {
332  const char *string;
333  int is_prefix;
334  int (*handler)(dir_connection_t *conn, const get_handler_args_t *args);
336 
337 static int handle_get_frontpage(dir_connection_t *conn,
338  const get_handler_args_t *args);
339 static int handle_get_current_consensus(dir_connection_t *conn,
340  const get_handler_args_t *args);
341 static int handle_get_status_vote(dir_connection_t *conn,
342  const get_handler_args_t *args);
343 static int handle_get_microdesc(dir_connection_t *conn,
344  const get_handler_args_t *args);
345 static int handle_get_descriptor(dir_connection_t *conn,
346  const get_handler_args_t *args);
347 static int handle_get_keys(dir_connection_t *conn,
348  const get_handler_args_t *args);
349 static int handle_get_hs_descriptor_v2(dir_connection_t *conn,
350  const get_handler_args_t *args);
351 static int handle_get_robots(dir_connection_t *conn,
352  const get_handler_args_t *args);
353 static int handle_get_networkstatus_bridges(dir_connection_t *conn,
354  const get_handler_args_t *args);
355 static int handle_get_next_bandwidth(dir_connection_t *conn,
356  const get_handler_args_t *args);
357 
359 static const url_table_ent_t url_table[] = {
360  { "/tor/", 0, handle_get_frontpage },
361  { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
362  { "/tor/status-vote/current/", 1, handle_get_status_vote },
363  { "/tor/status-vote/next/bandwidth", 0, handle_get_next_bandwidth },
364  { "/tor/status-vote/next/", 1, handle_get_status_vote },
365  { "/tor/micro/d/", 1, handle_get_microdesc },
366  { "/tor/server/", 1, handle_get_descriptor },
367  { "/tor/extra/", 1, handle_get_descriptor },
368  { "/tor/keys/", 1, handle_get_keys },
369  { "/tor/rendezvous2/", 1, handle_get_hs_descriptor_v2 },
370  { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 },
371  { "/tor/robots.txt", 0, handle_get_robots },
372  { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges },
373  { NULL, 0, NULL },
374 };
375 
382 MOCK_IMPL(STATIC int,
383 directory_handle_command_get,(dir_connection_t *conn, const char *headers,
384  const char *req_body, size_t req_body_len))
385 {
386  char *url, *url_mem, *header;
387  time_t if_modified_since = 0;
388  int zlib_compressed_in_url;
389  unsigned compression_methods_supported;
390 
391  /* We ignore the body of a GET request. */
392  (void)req_body;
393  (void)req_body_len;
394 
395  log_debug(LD_DIRSERV,"Received GET command.");
396 
397  conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
398 
399  if (parse_http_url(headers, &url) < 0) {
400  write_short_http_response(conn, 400, "Bad request");
401  return 0;
402  }
403  if ((header = http_get_header(headers, "If-Modified-Since: "))) {
404  struct tm tm;
405  if (parse_http_time(header, &tm) == 0) {
406  if (tor_timegm(&tm, &if_modified_since)<0) {
407  if_modified_since = 0;
408  } else {
409  log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header));
410  }
411  }
412  /* The correct behavior on a malformed If-Modified-Since header is to
413  * act as if no If-Modified-Since header had been given. */
414  tor_free(header);
415  }
416  log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
417 
418  url_mem = url;
419  {
420  size_t url_len = strlen(url);
421 
422  zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z");
423  if (zlib_compressed_in_url) {
424  url[url_len-2] = '\0';
425  }
426  }
427 
428  if ((header = http_get_header(headers, "Accept-Encoding: "))) {
429  compression_methods_supported = parse_accept_encoding_header(header);
430  tor_free(header);
431  } else {
432  compression_methods_supported = (1u << NO_METHOD);
433  }
434  if (zlib_compressed_in_url) {
435  compression_methods_supported |= (1u << ZLIB_METHOD);
436  }
437 
438  /* Remove all methods that we don't both support. */
439  compression_methods_supported &= tor_compress_get_supported_method_bitmask();
440 
441  get_handler_args_t args;
442  args.url = url;
443  args.headers = headers;
444  args.if_modified_since = if_modified_since;
445  args.compression_supported = compression_methods_supported;
446 
447  int i, result = -1;
448  for (i = 0; url_table[i].string; ++i) {
449  int match;
450  if (url_table[i].is_prefix) {
451  match = !strcmpstart(url, url_table[i].string);
452  } else {
453  match = !strcmp(url, url_table[i].string);
454  }
455  if (match) {
456  result = url_table[i].handler(conn, &args);
457  goto done;
458  }
459  }
460 
461  /* we didn't recognize the url */
462  write_short_http_response(conn, 404, "Not found");
463  result = 0;
464 
465  done:
466  tor_free(url_mem);
467  return result;
468 }
469 
472 static int
473 handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
474 {
475  (void) args; /* unused */
476  const char *frontpage = get_dirportfrontpage();
477 
478  if (frontpage) {
479  size_t dlen;
480  dlen = strlen(frontpage);
481  /* Let's return a disclaimer page (users shouldn't use V1 anymore,
482  and caches don't fetch '/', so this is safe). */
483 
484  /* [We don't check for write_bucket_low here, since we want to serve
485  * this page no matter what.] */
486  write_http_response_header_impl(conn, dlen, "text/html", "identity",
487  NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME);
488  connection_buf_add(frontpage, dlen, TO_CONN(conn));
489  } else {
490  write_short_http_response(conn, 404, "Not found");
491  }
492  return 0;
493 }
494 
500 static void
501 warn_consensus_is_not_reasonably_live(
502  const struct consensus_cache_entry_t *consensus,
503  const char *flavor, time_t now, bool is_too_new)
504 {
505 #define NOT_REASONABLY_LIVE_WARNING_INTERVAL (60*60)
506  static ratelim_t warned[2] = { RATELIM_INIT(
507  NOT_REASONABLY_LIVE_WARNING_INTERVAL),
508  RATELIM_INIT(
509  NOT_REASONABLY_LIVE_WARNING_INTERVAL) };
510  char timestamp[ISO_TIME_LEN+1];
511  /* valid_after if is_too_new, valid_until if !is_too_new */
512  time_t valid_time = 0;
513  char *dupes = NULL;
514 
515  if (is_too_new) {
516  if (consensus_cache_entry_get_valid_after(consensus, &valid_time))
517  return;
518  dupes = rate_limit_log(&warned[1], now);
519  } else {
520  if (consensus_cache_entry_get_valid_until(consensus, &valid_time))
521  return;
522  dupes = rate_limit_log(&warned[0], now);
523  }
524 
525  if (dupes) {
526  format_local_iso_time(timestamp, valid_time);
527  log_warn(LD_DIRSERV, "Our %s%sconsensus is too %s, so we will not "
528  "serve it to clients. It was valid %s %s local time and we "
529  "continued to serve it for up to 24 hours %s.%s",
530  flavor ? flavor : "",
531  flavor ? " " : "",
532  is_too_new ? "new" : "old",
533  is_too_new ? "after" : "until",
534  timestamp,
535  is_too_new ? "before it was valid" : "after it expired",
536  dupes);
537  tor_free(dupes);
538  }
539 }
540 
547 static int
548 parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location,
549  const char *action)
550 {
551  if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) ==
552  DIGEST256_LEN) {
553  return 0;
554  } else {
555  log_fn(LOG_PROTOCOL_WARN, LD_DIR,
556  "%s contained bogus digest %s; %s.",
557  location, escaped(hex), action);
558  return -1;
559  }
560 }
561 
566 static int
567 parse_or_diff_from_header(smartlist_t **digests_out, const char *headers)
568 {
569  char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER);
570  if (hdr == NULL) {
571  return -1;
572  }
573  smartlist_t *hex_digests = smartlist_new();
574  *digests_out = smartlist_new();
575  smartlist_split_string(hex_digests, hdr, " ",
576  SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
577  SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) {
578  uint8_t digest[DIGEST256_LEN];
579  if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header",
580  "ignoring")) {
581  smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest)));
582  }
583  } SMARTLIST_FOREACH_END(hex);
584  SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp));
585  smartlist_free(hex_digests);
586  tor_free(hdr);
587  return 0;
588 }
589 
595 #define FALLBACK_COMPRESS_METHOD ZLIB_METHOD
596 
604 static struct consensus_cache_entry_t *
605 find_best_diff(const smartlist_t *digests, int flav,
606  unsigned compression_methods,
607  compress_method_t *compression_used_out)
608 {
609  struct consensus_cache_entry_t *result = NULL;
610 
611  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
612  unsigned u;
613  for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
614  compress_method_t method = srv_meth_pref_precompressed[u];
615  if (0 == (compression_methods & (1u<<method)))
616  continue; // client doesn't like this one, or we don't have it.
617  if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256,
618  diff_from, DIGEST256_LEN,
619  method) == CONSDIFF_AVAILABLE) {
620  tor_assert_nonfatal(result);
621  *compression_used_out = method;
622  return result;
623  }
624  }
625  } SMARTLIST_FOREACH_END(diff_from);
626 
627  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
628  if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from,
629  DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
630  tor_assert_nonfatal(result);
631  *compression_used_out = FALLBACK_COMPRESS_METHOD;
632  return result;
633  }
634  } SMARTLIST_FOREACH_END(diff_from);
635 
636  return NULL;
637 }
638 
643 static struct consensus_cache_entry_t *
644 find_best_consensus(int flav,
645  unsigned compression_methods,
646  compress_method_t *compression_used_out)
647 {
648  struct consensus_cache_entry_t *result = NULL;
649  unsigned u;
650 
651  for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
652  compress_method_t method = srv_meth_pref_precompressed[u];
653 
654  if (0 == (compression_methods & (1u<<method)))
655  continue;
656 
657  if (consdiffmgr_find_consensus(&result, flav,
658  method) == CONSDIFF_AVAILABLE) {
659  tor_assert_nonfatal(result);
660  *compression_used_out = method;
661  return result;
662  }
663  }
664 
665  if (consdiffmgr_find_consensus(&result, flav,
666  FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
667  tor_assert_nonfatal(result);
668  *compression_used_out = FALLBACK_COMPRESS_METHOD;
669  return result;
670  }
671 
672  return NULL;
673 }
674 
678 static compress_method_t
679 find_best_compression_method(unsigned compression_methods, int stream)
680 {
681  unsigned u;
682  compress_method_t *methods;
683  size_t length;
684 
685  if (stream) {
686  methods = srv_meth_pref_streaming_compression;
687  length = ARRAY_LENGTH(srv_meth_pref_streaming_compression);
688  } else {
689  methods = srv_meth_pref_precompressed;
690  length = ARRAY_LENGTH(srv_meth_pref_precompressed);
691  }
692 
693  for (u = 0; u < length; ++u) {
694  compress_method_t method = methods[u];
695  if (compression_methods & (1u<<method))
696  return method;
697  }
698 
699  return NO_METHOD;
700 }
701 
704 static int
705 digest_list_contains_best_consensus(consensus_flavor_t flavor,
706  const smartlist_t *digests)
707 {
708  const networkstatus_t *ns = NULL;
709 
710  if (digests == NULL)
711  return 0;
712 
713  ns = networkstatus_get_latest_consensus_by_flavor(flavor);
714 
715  if (ns == NULL)
716  return 0;
717 
718  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, digest) {
720  return 1;
721  } SMARTLIST_FOREACH_END(digest);
722 
723  return 0;
724 }
725 
728 typedef struct {
730  char *flavor;
736  const char *want_fps;
744 
747 static void
748 parsed_consensus_request_clear(parsed_consensus_request_t *req)
749 {
750  if (!req)
751  return;
752  tor_free(req->flavor);
753  if (req->diff_from_digests) {
754  SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d));
755  smartlist_free(req->diff_from_digests);
756  }
757  memset(req, 0, sizeof(parsed_consensus_request_t));
758 }
759 
766 static int
767 parse_consensus_request(parsed_consensus_request_t *out,
768  const get_handler_args_t *args)
769 {
770  const char *url = args->url;
771  memset(out, 0, sizeof(parsed_consensus_request_t));
772  out->flav = FLAV_NS;
773 
774  const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/";
775  const char CONSENSUS_FLAVORED_PREFIX[] =
776  "/tor/status-vote/current/consensus-";
777 
778  /* figure out the flavor if any, and who we wanted to sign the thing */
779  const char *after_flavor = NULL;
780 
781  if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
782  const char *f, *cp;
783  f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
784  cp = strchr(f, '/');
785  if (cp) {
786  after_flavor = cp+1;
787  out->flavor = tor_strndup(f, cp-f);
788  } else {
789  out->flavor = tor_strdup(f);
790  }
791  int flav = networkstatus_parse_flavor_name(out->flavor);
792  if (flav < 0)
793  flav = FLAV_NS;
794  out->flav = flav;
795  } else {
796  if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
797  after_flavor = url+strlen(CONSENSUS_URL_PREFIX);
798  }
799 
800  /* see whether we've been asked explicitly for a diff from an older
801  * consensus. (The user might also have said that a diff would be okay,
802  * via X-Or-Diff-From-Consensus */
803  const char DIFF_COMPONENT[] = "diff/";
804  char *diff_hash_in_url = NULL;
805  if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) {
806  after_flavor += strlen(DIFF_COMPONENT);
807  const char *cp = strchr(after_flavor, '/');
808  if (cp) {
809  diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor);
810  out->want_fps = cp+1;
811  } else {
812  diff_hash_in_url = tor_strdup(after_flavor);
813  out->want_fps = NULL;
814  }
815  } else {
816  out->want_fps = after_flavor;
817  }
818 
819  if (diff_hash_in_url) {
820  uint8_t diff_from[DIGEST256_LEN];
821  out->diff_from_digests = smartlist_new();
822  out->diff_only = 1;
823  int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL",
824  "rejecting");
825  tor_free(diff_hash_in_url);
826  if (ok) {
828  tor_memdup(diff_from, DIGEST256_LEN));
829  } else {
830  return -1;
831  }
832  } else {
833  parse_or_diff_from_header(&out->diff_from_digests, args->headers);
834  }
835 
836  return 0;
837 }
838 
841 static int
842 handle_get_current_consensus(dir_connection_t *conn,
843  const get_handler_args_t *args)
844 {
845  const compress_method_t compress_method =
846  find_best_compression_method(args->compression_supported, 0);
847  const time_t if_modified_since = args->if_modified_since;
848  int clear_spool = 0;
849 
850  /* v3 network status fetch. */
851  long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
852 
853  time_t now = time(NULL);
855 
856  if (parse_consensus_request(&req, args) < 0) {
857  write_short_http_response(conn, 404, "Couldn't parse request");
858  goto done;
859  }
860 
861  if (digest_list_contains_best_consensus(req.flav,
862  req.diff_from_digests)) {
863  write_short_http_response(conn, 304, "Not modified");
865  goto done;
866  }
867 
868  struct consensus_cache_entry_t *cached_consensus = NULL;
869 
870  compress_method_t compression_used = NO_METHOD;
871  if (req.diff_from_digests) {
872  cached_consensus = find_best_diff(req.diff_from_digests, req.flav,
873  args->compression_supported,
874  &compression_used);
875  }
876 
877  if (req.diff_only && !cached_consensus) {
878  write_short_http_response(conn, 404, "No such diff available");
880  goto done;
881  }
882 
883  if (! cached_consensus) {
884  cached_consensus = find_best_consensus(req.flav,
885  args->compression_supported,
886  &compression_used);
887  }
888 
889  time_t valid_after, fresh_until, valid_until;
890  int have_valid_after = 0, have_fresh_until = 0, have_valid_until = 0;
891  if (cached_consensus) {
892  have_valid_after =
893  !consensus_cache_entry_get_valid_after(cached_consensus, &valid_after);
894  have_fresh_until =
895  !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
896  have_valid_until =
897  !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
898  }
899 
900  if (cached_consensus && have_valid_after &&
902  write_short_http_response(conn, 404, "Consensus is too new");
903  warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now,
904  1);
906  goto done;
907  } else if (
908  cached_consensus && have_valid_until &&
910  write_short_http_response(conn, 404, "Consensus is too old");
911  warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now,
912  0);
914  goto done;
915  }
916 
917  if (cached_consensus && req.want_fps &&
918  !client_likes_consensus(cached_consensus, req.want_fps)) {
919  write_short_http_response(conn, 404, "Consensus not signed by sufficient "
920  "number of requested authorities");
922  goto done;
923  }
924 
925  conn->spool = smartlist_new();
926  clear_spool = 1;
927  {
928  spooled_resource_t *spooled;
929  if (cached_consensus) {
930  spooled = spooled_resource_new_from_cache_entry(cached_consensus);
931  smartlist_add(conn->spool, spooled);
932  }
933  }
934 
935  lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0;
936 
937  size_t size_guess = 0;
938  int n_expired = 0;
939  dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since,
940  compress_method != NO_METHOD,
941  &size_guess,
942  &n_expired);
943 
944  if (!smartlist_len(conn->spool) && !n_expired) {
945  write_short_http_response(conn, 404, "Not found");
947  goto done;
948  } else if (!smartlist_len(conn->spool)) {
949  write_short_http_response(conn, 304, "Not modified");
951  goto done;
952  }
953 
954  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
955  log_debug(LD_DIRSERV,
956  "Client asked for network status lists, but we've been "
957  "writing too many bytes lately. Sending 503 Dir busy.");
958  write_short_http_response(conn, 503, "Directory busy, try again later");
960  goto done;
961  }
962 
963  tor_addr_t addr;
964  if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) {
966  &addr, NULL,
967  time(NULL));
969  /* Note that a request for a network status has started, so that we
970  * can measure the download time later on. */
971  if (conn->dirreq_id)
972  geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED);
973  else
974  geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess,
975  DIRREQ_DIRECT);
976  }
977 
978  /* Use this header to tell caches that the response depends on the
979  * X-Or-Diff-From-Consensus header (or lack thereof). */
980  const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n";
981 
982  clear_spool = 0;
983 
984  // The compress_method might have been NO_METHOD, but we store the data
985  // compressed. Decompress them using `compression_used`. See fallback code in
986  // find_best_consensus() and find_best_diff().
987  write_http_response_headers(conn, -1,
988  compress_method == NO_METHOD ?
989  NO_METHOD : compression_used,
990  vary_header,
991  smartlist_len(conn->spool) == 1 ? lifetime : 0);
992 
993  if (compress_method == NO_METHOD && smartlist_len(conn->spool))
994  conn->compress_state = tor_compress_new(0, compression_used,
995  HIGH_COMPRESSION);
996 
997  /* Prime the connection with some data. */
998  const int initial_flush_result = connection_dirserv_flushed_some(conn);
999  tor_assert_nonfatal(initial_flush_result == 0);
1000  goto done;
1001 
1002  done:
1003  parsed_consensus_request_clear(&req);
1004  if (clear_spool) {
1005  dir_conn_clear_spool(conn);
1006  }
1007  return 0;
1008 }
1009 
1012 static int
1013 handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
1014 {
1015  const char *url = args->url;
1016  {
1017  ssize_t body_len = 0;
1018  ssize_t estimated_len = 0;
1019  int lifetime = 60; /* XXXX?? should actually use vote intervals. */
1020  /* This smartlist holds strings that we can compress on the fly. */
1021  smartlist_t *items = smartlist_new();
1022  /* This smartlist holds cached_dir_t objects that have a precompressed
1023  * deflated version. */
1024  smartlist_t *dir_items = smartlist_new();
1025  dirvote_dirreq_get_status_vote(url, items, dir_items);
1026  if (!smartlist_len(dir_items) && !smartlist_len(items)) {
1027  write_short_http_response(conn, 404, "Not found");
1028  goto vote_done;
1029  }
1030 
1031  /* We're sending items from at most one kind of source */
1032  tor_assert_nonfatal(smartlist_len(items) == 0 ||
1033  smartlist_len(dir_items) == 0);
1034 
1035  int streaming;
1036  unsigned mask;
1037  if (smartlist_len(items)) {
1038  /* We're taking strings and compressing them on the fly. */
1039  streaming = 1;
1040  mask = ~0u;
1041  } else {
1042  /* We're taking cached_dir_t objects. We only have them uncompressed
1043  * or deflated. */
1044  streaming = 0;
1045  mask = (1u<<NO_METHOD) | (1u<<ZLIB_METHOD);
1046  }
1047  const compress_method_t compress_method = find_best_compression_method(
1048  args->compression_supported&mask, streaming);
1049 
1050  SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
1051  body_len += compress_method != NO_METHOD ?
1052  d->dir_compressed_len : d->dir_len);
1053  estimated_len += body_len;
1054  SMARTLIST_FOREACH(items, const char *, item, {
1055  size_t ln = strlen(item);
1056  if (compress_method != NO_METHOD) {
1057  estimated_len += ln/2;
1058  } else {
1059  body_len += ln; estimated_len += ln;
1060  }
1061  });
1062 
1063  if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
1064  write_short_http_response(conn, 503, "Directory busy, try again later");
1065  goto vote_done;
1066  }
1067  write_http_response_header(conn, body_len ? body_len : -1,
1068  compress_method,
1069  lifetime);
1070 
1071  if (smartlist_len(items)) {
1072  if (compress_method != NO_METHOD) {
1073  conn->compress_state = tor_compress_new(1, compress_method,
1074  choose_compression_level(estimated_len));
1075  }
1076 
1077  SMARTLIST_FOREACH(items, const char *, c,
1078  connection_dir_buf_add(c, strlen(c), conn,
1079  c_sl_idx == c_sl_len - 1));
1080  } else {
1081  SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
1082  connection_buf_add(compress_method != NO_METHOD ?
1083  d->dir_compressed : d->dir,
1084  compress_method != NO_METHOD ?
1085  d->dir_compressed_len : d->dir_len,
1086  TO_CONN(conn)));
1087  }
1088  vote_done:
1089  smartlist_free(items);
1090  smartlist_free(dir_items);
1091  goto done;
1092  }
1093  done:
1094  return 0;
1095 }
1096 
1099 static int
1100 handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
1101 {
1102  const char *url = args->url;
1103  const compress_method_t compress_method =
1104  find_best_compression_method(args->compression_supported, 1);
1105  int clear_spool = 1;
1106  {
1107  conn->spool = smartlist_new();
1108 
1109  dir_split_resource_into_spoolable(url+strlen("/tor/micro/d/"),
1110  DIR_SPOOL_MICRODESC,
1111  conn->spool, NULL,
1112  DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ);
1113 
1114  size_t size_guess = 0;
1116  compress_method != NO_METHOD,
1117  &size_guess, NULL);
1118  if (smartlist_len(conn->spool) == 0) {
1119  write_short_http_response(conn, 404, "Not found");
1120  goto done;
1121  }
1122  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
1123  log_info(LD_DIRSERV,
1124  "Client asked for server descriptors, but we've been "
1125  "writing too many bytes lately. Sending 503 Dir busy.");
1126  write_short_http_response(conn, 503, "Directory busy, try again later");
1127  goto done;
1128  }
1129 
1130  clear_spool = 0;
1131  write_http_response_header(conn, -1,
1132  compress_method,
1133  MICRODESC_CACHE_LIFETIME);
1134 
1135  if (compress_method != NO_METHOD)
1136  conn->compress_state = tor_compress_new(1, compress_method,
1137  choose_compression_level(size_guess));
1138 
1139  const int initial_flush_result = connection_dirserv_flushed_some(conn);
1140  tor_assert_nonfatal(initial_flush_result == 0);
1141  goto done;
1142  }
1143 
1144  done:
1145  if (clear_spool) {
1146  dir_conn_clear_spool(conn);
1147  }
1148  return 0;
1149 }
1150 
1153 static int
1154 handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
1155 {
1156  const char *url = args->url;
1157  const compress_method_t compress_method =
1158  find_best_compression_method(args->compression_supported, 1);
1159  const or_options_t *options = get_options();
1160  int clear_spool = 1;
1161  if (!strcmpstart(url,"/tor/server/") ||
1162  (!options->BridgeAuthoritativeDir &&
1163  !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) {
1164  int res;
1165  const char *msg = NULL;
1166  int cache_lifetime = 0;
1167  int is_extra = !strcmpstart(url,"/tor/extra/");
1168  url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/");
1169  dir_spool_source_t source;
1170  time_t publish_cutoff = 0;
1171  if (!strcmpstart(url, "d/")) {
1172  source =
1173  is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST;
1174  } else {
1175  source =
1176  is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP;
1177  /* We only want to apply a publish cutoff when we're requesting
1178  * resources by fingerprint. */
1179  publish_cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
1180  }
1181 
1182  conn->spool = smartlist_new();
1183  res = dirserv_get_routerdesc_spool(conn->spool, url,
1184  source,
1186  &msg);
1187 
1188  if (!strcmpstart(url, "all")) {
1189  cache_lifetime = FULL_DIR_CACHE_LIFETIME;
1190  } else if (smartlist_len(conn->spool) == 1) {
1191  cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME;
1192  }
1193 
1194  size_t size_guess = 0;
1195  int n_expired = 0;
1196  dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff,
1197  compress_method != NO_METHOD,
1198  &size_guess, &n_expired);
1199 
1200  /* If we are the bridge authority and the descriptor is a bridge
1201  * descriptor, remember that we served this descriptor for desc stats. */
1202  /* XXXX it's a bit of a kludge to have this here. */
1203  if (get_options()->BridgeAuthoritativeDir &&
1204  source == DIR_SPOOL_SERVER_BY_FP) {
1205  SMARTLIST_FOREACH_BEGIN(conn->spool, spooled_resource_t *, spooled) {
1206  const routerinfo_t *router =
1207  router_get_by_id_digest((const char *)spooled->digest);
1208  /* router can be NULL here when the bridge auth is asked for its own
1209  * descriptor. */
1210  if (router && router->purpose == ROUTER_PURPOSE_BRIDGE)
1211  rep_hist_note_desc_served(router->cache_info.identity_digest);
1212  } SMARTLIST_FOREACH_END(spooled);
1213  }
1214 
1215  if (res < 0 || size_guess == 0 || smartlist_len(conn->spool) == 0) {
1216  if (msg == NULL)
1217  msg = "Not found";
1218  write_short_http_response(conn, 404, msg);
1219  } else {
1220  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
1221  log_info(LD_DIRSERV,
1222  "Client asked for server descriptors, but we've been "
1223  "writing too many bytes lately. Sending 503 Dir busy.");
1224  write_short_http_response(conn, 503,
1225  "Directory busy, try again later");
1226  dir_conn_clear_spool(conn);
1227  goto done;
1228  }
1229  write_http_response_header(conn, -1, compress_method, cache_lifetime);
1230  if (compress_method != NO_METHOD)
1231  conn->compress_state = tor_compress_new(1, compress_method,
1232  choose_compression_level(size_guess));
1233  clear_spool = 0;
1234  /* Prime the connection with some data. */
1235  int initial_flush_result = connection_dirserv_flushed_some(conn);
1236  tor_assert_nonfatal(initial_flush_result == 0);
1237  }
1238  goto done;
1239  }
1240  done:
1241  if (clear_spool)
1242  dir_conn_clear_spool(conn);
1243  return 0;
1244 }
1245 
1248 static int
1249 handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
1250 {
1251  const char *url = args->url;
1252  const compress_method_t compress_method =
1253  find_best_compression_method(args->compression_supported, 1);
1254  const time_t if_modified_since = args->if_modified_since;
1255  {
1256  smartlist_t *certs = smartlist_new();
1257  ssize_t len = -1;
1258  if (!strcmp(url, "/tor/keys/all")) {
1259  authority_cert_get_all(certs);
1260  } else if (!strcmp(url, "/tor/keys/authority")) {
1261  authority_cert_t *cert = get_my_v3_authority_cert();
1262  if (cert)
1263  smartlist_add(certs, cert);
1264  } else if (!strcmpstart(url, "/tor/keys/fp/")) {
1265  smartlist_t *fps = smartlist_new();
1266  dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"),
1267  fps, NULL,
1268  DSR_HEX|DSR_SORT_UNIQ);
1269  SMARTLIST_FOREACH(fps, char *, d, {
1271  if (c) smartlist_add(certs, c);
1272  tor_free(d);
1273  });
1274  smartlist_free(fps);
1275  } else if (!strcmpstart(url, "/tor/keys/sk/")) {
1276  smartlist_t *fps = smartlist_new();
1277  dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"),
1278  fps, NULL,
1279  DSR_HEX|DSR_SORT_UNIQ);
1280  SMARTLIST_FOREACH(fps, char *, d, {
1282  if (c) smartlist_add(certs, c);
1283  tor_free(d);
1284  });
1285  smartlist_free(fps);
1286  } else if (!strcmpstart(url, "/tor/keys/fp-sk/")) {
1287  smartlist_t *fp_sks = smartlist_new();
1288  dir_split_resource_into_fingerprint_pairs(url+strlen("/tor/keys/fp-sk/"),
1289  fp_sks);
1290  SMARTLIST_FOREACH(fp_sks, fp_pair_t *, pair, {
1292  pair->second);
1293  if (c) smartlist_add(certs, c);
1294  tor_free(pair);
1295  });
1296  smartlist_free(fp_sks);
1297  } else {
1298  write_short_http_response(conn, 400, "Bad request");
1299  goto keys_done;
1300  }
1301  if (!smartlist_len(certs)) {
1302  write_short_http_response(conn, 404, "Not found");
1303  goto keys_done;
1304  }
1305  SMARTLIST_FOREACH(certs, authority_cert_t *, c,
1306  if (c->cache_info.published_on < if_modified_since)
1307  SMARTLIST_DEL_CURRENT(certs, c));
1308  if (!smartlist_len(certs)) {
1309  write_short_http_response(conn, 304, "Not modified");
1310  goto keys_done;
1311  }
1312  len = 0;
1313  SMARTLIST_FOREACH(certs, authority_cert_t *, c,
1314  len += c->cache_info.signed_descriptor_len);
1315 
1316  if (global_write_bucket_low(TO_CONN(conn),
1317  compress_method != NO_METHOD ? len/2 : len,
1318  2)) {
1319  write_short_http_response(conn, 503, "Directory busy, try again later");
1320  goto keys_done;
1321  }
1322 
1323  write_http_response_header(conn,
1324  compress_method != NO_METHOD ? -1 : len,
1325  compress_method,
1326  60*60);
1327  if (compress_method != NO_METHOD) {
1328  conn->compress_state = tor_compress_new(1, compress_method,
1329  choose_compression_level(len));
1330  }
1331 
1332  SMARTLIST_FOREACH(certs, authority_cert_t *, c,
1333  connection_dir_buf_add(c->cache_info.signed_descriptor_body,
1334  c->cache_info.signed_descriptor_len,
1335  conn, c_sl_idx == c_sl_len - 1));
1336  keys_done:
1337  smartlist_free(certs);
1338  goto done;
1339  }
1340  done:
1341  return 0;
1342 }
1343 
1346 static int
1347 handle_get_hs_descriptor_v2(dir_connection_t *conn,
1348  const get_handler_args_t *args)
1349 {
1350  const char *url = args->url;
1351  if (connection_dir_is_encrypted(conn)) {
1352  /* Handle v2 rendezvous descriptor fetch request. */
1353  const char *descp;
1354  const char *query = url + strlen("/tor/rendezvous2/");
1355  if (rend_valid_descriptor_id(query)) {
1356  log_info(LD_REND, "Got a v2 rendezvous descriptor request for ID '%s'",
1357  safe_str(escaped(query)));
1358  switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) {
1359  case 1: /* valid */
1360  write_http_response_header(conn, strlen(descp), NO_METHOD, 0);
1361  connection_buf_add(descp, strlen(descp), TO_CONN(conn));
1362  break;
1363  case 0: /* well-formed but not present */
1364  write_short_http_response(conn, 404, "Not found");
1365  break;
1366  case -1: /* not well-formed */
1367  write_short_http_response(conn, 400, "Bad request");
1368  break;
1369  }
1370  } else { /* not well-formed */
1371  write_short_http_response(conn, 400, "Bad request");
1372  }
1373  goto done;
1374  } else {
1375  /* Not encrypted! */
1376  write_short_http_response(conn, 404, "Not found");
1377  }
1378  done:
1379  return 0;
1380 }
1381 
1384 STATIC int
1385 handle_get_hs_descriptor_v3(dir_connection_t *conn,
1386  const get_handler_args_t *args)
1387 {
1388  int retval;
1389  const char *desc_str = NULL;
1390  const char *pubkey_str = NULL;
1391  const char *url = args->url;
1392 
1393  /* Reject unencrypted dir connections */
1394  if (!connection_dir_is_encrypted(conn)) {
1395  write_short_http_response(conn, 404, "Not found");
1396  goto done;
1397  }
1398 
1399  /* After the path prefix follows the base64 encoded blinded pubkey which we
1400  * use to get the descriptor from the cache. Skip the prefix and get the
1401  * pubkey. */
1402  tor_assert(!strcmpstart(url, "/tor/hs/3/"));
1403  pubkey_str = url + strlen("/tor/hs/3/");
1404  retval = hs_cache_lookup_as_dir(HS_VERSION_THREE,
1405  pubkey_str, &desc_str);
1406  if (retval <= 0 || desc_str == NULL) {
1407  write_short_http_response(conn, 404, "Not found");
1408  goto done;
1409  }
1410 
1411  /* Found requested descriptor! Pass it to this nice client. */
1412  write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0);
1413  connection_buf_add(desc_str, strlen(desc_str), TO_CONN(conn));
1414 
1415  done:
1416  return 0;
1417 }
1418 
1421 static int
1422 handle_get_networkstatus_bridges(dir_connection_t *conn,
1423  const get_handler_args_t *args)
1424 {
1425  const char *headers = args->headers;
1426 
1427  const or_options_t *options = get_options();
1428  if (options->BridgeAuthoritativeDir &&
1429  options->BridgePassword_AuthDigest_ &&
1431  char *status;
1432  char digest[DIGEST256_LEN];
1433 
1434  char *header = http_get_header(headers, "Authorization: Basic ");
1435  if (header)
1436  crypto_digest256(digest, header, strlen(header), DIGEST_SHA256);
1437 
1438  /* now make sure the password is there and right */
1439  if (!header ||
1440  tor_memneq(digest,
1442  write_short_http_response(conn, 404, "Not found");
1443  tor_free(header);
1444  goto done;
1445  }
1446  tor_free(header);
1447 
1448  /* all happy now. send an answer. */
1449  status = networkstatus_getinfo_by_purpose("bridge", time(NULL));
1450  size_t dlen = strlen(status);
1451  write_http_response_header(conn, dlen, NO_METHOD, 0);
1452  connection_buf_add(status, dlen, TO_CONN(conn));
1453  tor_free(status);
1454  goto done;
1455  }
1456  done:
1457  return 0;
1458 }
1459 
1461 static int
1462 handle_get_next_bandwidth(dir_connection_t *conn,
1463  const get_handler_args_t *args)
1464 {
1465  log_debug(LD_DIR, "Getting next bandwidth.");
1466  const or_options_t *options = get_options();
1467  const compress_method_t compress_method =
1468  find_best_compression_method(args->compression_supported, 1);
1469 
1470  if (options->V3BandwidthsFile) {
1471  char *bandwidth = read_file_to_str(options->V3BandwidthsFile,
1472  RFTS_IGNORE_MISSING, NULL);
1473  if (bandwidth != NULL) {
1474  ssize_t len = strlen(bandwidth);
1475  write_http_response_header(conn, compress_method != NO_METHOD ? -1 : len,
1476  compress_method, BANDWIDTH_CACHE_LIFETIME);
1477  if (compress_method != NO_METHOD) {
1478  conn->compress_state = tor_compress_new(1, compress_method,
1479  choose_compression_level(len/2));
1480  log_debug(LD_DIR, "Compressing bandwidth file.");
1481  } else {
1482  log_debug(LD_DIR, "Not compressing bandwidth file.");
1483  }
1484  connection_dir_buf_add((const char*)bandwidth, len, conn, 1);
1485  tor_free(bandwidth);
1486  return 0;
1487  }
1488  }
1489  write_short_http_response(conn, 404, "Not found");
1490  return 0;
1491 }
1492 
1494 static int
1495 handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
1496 {
1497  (void)args;
1498  {
1499  const char robots[] = "User-agent: *\r\nDisallow: /\r\n";
1500  size_t len = strlen(robots);
1501  write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME);
1502  connection_buf_add(robots, len, TO_CONN(conn));
1503  }
1504  return 0;
1505 }
1506 
1507 /* Given the <b>url</b> from a POST request, try to extract the version number
1508  * using the provided <b>prefix</b>. The version should be after the prefix and
1509  * ending with the separator "/". For instance:
1510  * /tor/hs/3/publish
1511  *
1512  * On success, <b>end_pos</b> points to the position right after the version
1513  * was found. On error, it is set to NULL.
1514  *
1515  * Return version on success else negative value. */
1516 STATIC int
1517 parse_hs_version_from_post(const char *url, const char *prefix,
1518  const char **end_pos)
1519 {
1520  int ok;
1521  unsigned long version;
1522  const char *start;
1523  char *end = NULL;
1524 
1525  tor_assert(url);
1526  tor_assert(prefix);
1527  tor_assert(end_pos);
1528 
1529  /* Check if the prefix does start the url. */
1530  if (strcmpstart(url, prefix)) {
1531  goto err;
1532  }
1533  /* Move pointer to the end of the prefix string. */
1534  start = url + strlen(prefix);
1535  /* Try this to be the HS version and if we are still at the separator, next
1536  * will be move to the right value. */
1537  version = tor_parse_long(start, 10, 0, INT_MAX, &ok, &end);
1538  if (!ok) {
1539  goto err;
1540  }
1541 
1542  *end_pos = end;
1543  return (int) version;
1544  err:
1545  *end_pos = NULL;
1546  return -1;
1547 }
1548 
1549 /* Handle the POST request for a hidden service descripror. The request is in
1550  * <b>url</b>, the body of the request is in <b>body</b>. Return 200 on success
1551  * else return 400 indicating a bad request. */
1552 STATIC int
1553 handle_post_hs_descriptor(const char *url, const char *body)
1554 {
1555  int version;
1556  const char *end_pos;
1557 
1558  tor_assert(url);
1559  tor_assert(body);
1560 
1561  version = parse_hs_version_from_post(url, "/tor/hs/", &end_pos);
1562  if (version < 0) {
1563  goto err;
1564  }
1565 
1566  /* We have a valid version number, now make sure it's a publish request. Use
1567  * the end position just after the version and check for the command. */
1568  if (strcmpstart(end_pos, "/publish")) {
1569  goto err;
1570  }
1571 
1572  switch (version) {
1573  case HS_VERSION_THREE:
1574  if (hs_cache_store_as_dir(body) < 0) {
1575  goto err;
1576  }
1577  log_info(LD_REND, "Publish request for HS descriptor handled "
1578  "successfully.");
1579  break;
1580  default:
1581  /* Unsupported version, return a bad request. */
1582  goto err;
1583  }
1584 
1585  return 200;
1586  err:
1587  /* Bad request. */
1588  return 400;
1589 }
1590 
1596 MOCK_IMPL(STATIC int,
1597 directory_handle_command_post,(dir_connection_t *conn, const char *headers,
1598  const char *body, size_t body_len))
1599 {
1600  char *url = NULL;
1601  const or_options_t *options = get_options();
1602 
1603  log_debug(LD_DIRSERV,"Received POST command.");
1604 
1605  conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
1606 
1607  if (!public_server_mode(options)) {
1608  log_info(LD_DIR, "Rejected dir post request from %s "
1609  "since we're not a public relay.", conn->base_.address);
1610  write_short_http_response(conn, 503, "Not acting as a public relay");
1611  goto done;
1612  }
1613 
1614  if (parse_http_url(headers, &url) < 0) {
1615  write_short_http_response(conn, 400, "Bad request");
1616  return 0;
1617  }
1618  log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
1619 
1620  /* Handle v2 rendezvous service publish request. */
1621  if (connection_dir_is_encrypted(conn) &&
1622  !strcmpstart(url,"/tor/rendezvous2/publish")) {
1624  log_warn(LD_REND, "Rejected v2 rend descriptor (body size %d) from %s.",
1625  (int)body_len, conn->base_.address);
1626  write_short_http_response(conn, 400,
1627  "Invalid v2 service descriptor rejected");
1628  } else {
1629  write_short_http_response(conn, 200, "Service descriptor (v2) stored");
1630  log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted");
1631  }
1632  goto done;
1633  }
1634 
1635  /* Handle HS descriptor publish request. */
1636  /* XXX: This should be disabled with a consensus param until we want to
1637  * the prop224 be deployed and thus use. */
1638  if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) {
1639  const char *msg = "HS descriptor stored successfully.";
1640 
1641  /* We most probably have a publish request for an HS descriptor. */
1642  int code = handle_post_hs_descriptor(url, body);
1643  if (code != 200) {
1644  msg = "Invalid HS descriptor. Rejected.";
1645  }
1646  write_short_http_response(conn, code, msg);
1647  goto done;
1648  }
1649 
1650  if (!authdir_mode(options)) {
1651  /* we just provide cached directories; we don't want to
1652  * receive anything. */
1653  write_short_http_response(conn, 400, "Nonauthoritative directory does not "
1654  "accept posted server descriptors");
1655  goto done;
1656  }
1657 
1658  if (authdir_mode(options) &&
1659  !strcmp(url,"/tor/")) { /* server descriptor post */
1660  const char *msg = "[None]";
1661  uint8_t purpose = authdir_mode_bridge(options) ?
1662  ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
1664  purpose, conn->base_.address, &msg);
1665  tor_assert(msg);
1666 
1667  if (r == ROUTER_ADDED_SUCCESSFULLY) {
1668  write_short_http_response(conn, 200, msg);
1669  } else if (WRA_WAS_OUTDATED(r)) {
1670  write_http_response_header_impl(conn, -1, NULL, NULL,
1671  "X-Descriptor-Not-New: Yes\r\n", -1);
1672  } else {
1673  log_info(LD_DIRSERV,
1674  "Rejected router descriptor or extra-info from %s "
1675  "(\"%s\").",
1676  conn->base_.address, msg);
1677  write_short_http_response(conn, 400, msg);
1678  }
1679  goto done;
1680  }
1681 
1682  if (authdir_mode_v3(options) &&
1683  !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */
1684  const char *msg = "OK";
1685  int status;
1686  if (dirvote_add_vote(body, &msg, &status)) {
1687  write_short_http_response(conn, status, "Vote stored");
1688  } else {
1689  tor_assert(msg);
1690  log_warn(LD_DIRSERV, "Rejected vote from %s (\"%s\").",
1691  conn->base_.address, msg);
1692  write_short_http_response(conn, status, msg);
1693  }
1694  goto done;
1695  }
1696 
1697  if (authdir_mode_v3(options) &&
1698  !strcmp(url,"/tor/post/consensus-signature")) { /* sigs on consensus. */
1699  const char *msg = NULL;
1700  if (dirvote_add_signatures(body, conn->base_.address, &msg)>=0) {
1701  write_short_http_response(conn, 200, msg?msg:"Signatures stored");
1702  } else {
1703  log_warn(LD_DIR, "Unable to store signatures posted by %s: %s",
1704  conn->base_.address, msg?msg:"???");
1705  write_short_http_response(conn, 400,
1706  msg?msg:"Unable to store signatures");
1707  }
1708  goto done;
1709  }
1710 
1711  /* we didn't recognize the url */
1712  write_short_http_response(conn, 404, "Not found");
1713 
1714  done:
1715  tor_free(url);
1716  return 0;
1717 }
1718 
1722 static void
1723 http_set_address_origin(const char *headers, connection_t *conn)
1724 {
1725  char *fwd;
1726 
1727  fwd = http_get_header(headers, "Forwarded-For: ");
1728  if (!fwd)
1729  fwd = http_get_header(headers, "X-Forwarded-For: ");
1730  if (fwd) {
1731  tor_addr_t toraddr;
1732  if (tor_addr_parse(&toraddr,fwd) == -1 ||
1733  tor_addr_is_internal(&toraddr,0)) {
1734  log_debug(LD_DIR, "Ignoring local/internal IP %s", escaped(fwd));
1735  tor_free(fwd);
1736  return;
1737  }
1738 
1739  tor_free(conn->address);
1740  conn->address = tor_strdup(fwd);
1741  tor_free(fwd);
1742  }
1743 }
1744 
1750 int
1752 {
1753  char *headers=NULL, *body=NULL;
1754  size_t body_len=0;
1755  int r;
1756 
1757  tor_assert(conn);
1758  tor_assert(conn->base_.type == CONN_TYPE_DIR);
1759 
1761  &headers, MAX_HEADERS_SIZE,
1762  &body, &body_len, MAX_DIR_UL_SIZE, 0)) {
1763  case -1: /* overflow */
1764  log_warn(LD_DIRSERV,
1765  "Request too large from address '%s' to DirPort. Closing.",
1766  safe_str(conn->base_.address));
1767  return -1;
1768  case 0:
1769  log_debug(LD_DIRSERV,"command not all here yet.");
1770  return 0;
1771  /* case 1, fall through */
1772  }
1773 
1774  http_set_address_origin(headers, TO_CONN(conn));
1775  // we should escape headers here as well,
1776  // but we can't call escaped() twice, as it uses the same buffer
1777  //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, escaped(body));
1778 
1779  if (!strncasecmp(headers,"GET",3))
1780  r = directory_handle_command_get(conn, headers, body, body_len);
1781  else if (!strncasecmp(headers,"POST",4))
1782  r = directory_handle_command_post(conn, headers, body, body_len);
1783  else {
1784  log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
1785  "Got headers %s with unknown command. Closing.",
1786  escaped(headers));
1787  r = -1;
1788  }
1789 
1790  tor_free(headers); tor_free(body);
1791  return r;
1792 }
Header file for dirserv.c.
tor_compress_state_t * tor_compress_new(int compress, compress_method_t method, compression_level_t compression_level)
Definition: compress.c:481
void dir_conn_clear_spool(dir_connection_t *conn)
Definition: dirserv.c:899
void geoip_note_client_seen(geoip_client_action_t action, const tor_addr_t *addr, const char *transport_name, time_t now)
Definition: geoip_stats.c:227
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
#define TO_CONN(c)
Definition: or.h:735
int rend_valid_descriptor_id(const char *query)
Definition: rendcommon.c:733
was_router_added_t
Definition: routerlist.h:17
Header file for geoip_stats.c.
char * V3BandwidthsFile
Header file for dircache.c.
tor_addr_t addr
int tor_timegm(const struct tm *tm, time_t *time_out)
Definition: time_fmt.c:96
void format_local_iso_time(char *buf, time_t t)
Definition: time_fmt.c:285
Header file for connection.c.
size_t dir_compressed_len
Definition: cached_dir_st.h:16
unsigned compression_supported
Definition: dircache.c:311
int directory_handle_command(dir_connection_t *conn)
Definition: dircache.c:1751
uint8_t state
Definition: connection_st.h:44
int authdir_mode(const or_options_t *options)
Definition: authmode.c:25
Headers for compress.c.
smartlist_t * diff_from_digests
Definition: dircache.c:739
char * networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
int rend_cache_lookup_v2_desc_as_dir(const char *desc_id, const char **desc)
Definition: rendcache.c:592
int dir_split_resource_into_fingerprint_pairs(const char *res, smartlist_t *pairs_out)
Definition: directory.c:485
Header file for directory.c.
void smartlist_add(smartlist_t *sl, void *element)
int parse_http_time(const char *date, struct tm *tm)
Definition: time_fmt.c:409
int dirserv_get_routerdesc_spool(smartlist_t *spool_out, const char *key, dir_spool_source_t source, int conn_is_encrypted, const char **msg_out)
Definition: dirserv.c:276
#define MAX_HEADERS_SIZE
Definition: or.h:123
char * rate_limit_log(ratelim_t *lim, time_t now)
Definition: ratelim.c:41
smartlist_t * spool
int authdir_mode_bridge(const or_options_t *options)
Definition: authmode.c:67
Header file for config.c.
Header file for authcert.c.
uint8_t digest[DIGEST256_LEN]
Definition: dirserv.h:55
uint8_t digest_sha3_as_signed[DIGEST256_LEN]
const char * headers
Definition: dircache.c:318
void rep_hist_note_desc_served(const char *desc)
Definition: rephist.c:2200
struct tor_compress_state_t * compress_state
unsigned tor_compress_get_supported_method_bitmask(void)
Definition: compress.c:328
int strcmpstart(const char *s1, const char *s2)
Definition: util_string.c:206
#define tor_free(p)
Definition: malloc.h:52
void authority_cert_get_all(smartlist_t *certs_out)
Definition: authcert.c:666
int connection_fetch_from_buf_http(connection_t *conn, char **headers_out, size_t max_headerlen, char **body_out, size_t *body_used, size_t max_bodylen, int force_complete)
Definition: connection.c:3901
const uint8_t * body
Definition: conscache.c:51
int networkstatus_valid_until_is_reasonably_live(time_t valid_until, time_t now)
Definition: conscache.c:27
#define SMARTLIST_DEL_CURRENT(sl, var)
int connection_dirserv_flushed_some(dir_connection_t *conn)
Definition: dirserv.c:854
int global_write_bucket_low(connection_t *conn, size_t attempt, int priority)
Definition: connection.c:3203
pending_vote_t * dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
Definition: dirvote.c:3131
const char * url
Definition: dircache.c:316
signed_descriptor_t cache_info
spooled_resource_t * spooled_resource_new_from_cache_entry(consensus_cache_entry_t *entry)
Definition: dirserv.c:457
Header file for directory authority mode.
int networkstatus_parse_flavor_name(const char *flavname)
#define DIGEST256_LEN
Definition: digest_sizes.h:23
consensus_flavor_t flav
Definition: dircache.c:732
int rend_cache_store_v2_desc_as_dir(const char *desc)
Definition: rendcache.c:627
const char * compression_method_get_name(compress_method_t method)
Definition: compress.c:364
tor_assert(buffer)
int networkstatus_valid_after_is_reasonably_live(time_t valid_after, time_t now)
Header file for routermode.c.
int tor_memeq(const void *a, const void *b, size_t sz)
Definition: di_ops.c:107
int tor_asprintf(char **strp, const char *fmt,...)
Definition: printf.c:75
const routerinfo_t * router_get_by_id_digest(const char *digest)
Definition: routerlist.c:690
authority_cert_t * authority_cert_get_newest_by_id(const char *id_digest)
Definition: authcert.c:603
dir_spool_source_t
Definition: dirserv.h:20
Header file for rendcache.c.
Master header file for Tor-specific functionality.
#define LD_DIRSERV
Definition: log.h:87
int tor_addr_parse(tor_addr_t *addr, const char *src)
Definition: address.c:1255
void buf_add_string(buf_t *buf, const char *string)
Definition: buffers.c:559
Header file for rephist.c.
int have_been_under_memory_pressure(void)
Definition: relay.c:2723
unsigned int type
Definition: connection_st.h:45
time_t if_modified_since
Definition: dircache.c:314
void geoip_note_ns_response(geoip_ns_response_t response)
Definition: geoip_stats.c:393
authority_cert_t * authority_cert_get_by_sk_digest(const char *sk_digest)
Definition: authcert.c:623
#define RFTS_IGNORE_MISSING
Definition: files.h:97
#define LD_REND
Definition: log.h:81
int BridgeAuthoritativeDir
Header file for process_descs.c.
void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, dirreq_type_t type)
Definition: geoip_stats.c:529
void format_rfc1123_time(char *buf, time_t t)
Definition: time_fmt.c:182
void connection_dir_buf_add(const char *string, size_t len, dir_connection_t *dir_conn, int done)
Definition: connection.c:4366
#define LD_DIR
Definition: log.h:85
void connection_buf_add_buf(connection_t *conn, buf_t *buf)
Definition: connection.c:4390
char * http_get_header(const char *headers, const char *which)
Definition: directory.c:229
authority_cert_t * authority_cert_get_by_digests(const char *id_digest, const char *sk_digest)
Definition: authcert.c:650
char identity_digest[DIGEST_LEN]
Header file for fp_pair.c.
#define ROUTER_MAX_AGE_TO_PUBLISH
Definition: or.h:162
char * BridgePassword_AuthDigest_
#define IF_BUG_ONCE(cond)
Definition: util_bug.h:234
void buf_add_printf(buf_t *buf, const char *format,...)
Definition: buffers.c:566
Header file for dirvote.c.
Header file for relay.c.
#define SMARTLIST_FOREACH(sl, type, var, cmd)
int connection_dir_is_encrypted(const dir_connection_t *conn)
Definition: directory.c:160
const char * escaped(const char *s)
Definition: escape.c:126
void dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn, time_t cutoff, int compression, size_t *size_out, int *n_expired_out)
Definition: dirserv.c:756
consensus_flavor_t
Definition: or.h:863
#define ARRAY_LENGTH(x)
#define log_fn(severity, domain, args,...)
Definition: log.h:272
int dir_split_resource_into_fingerprints(const char *resource, smartlist_t *fp_out, int *compressed_out, int flags)
Definition: directory.c:544
time_t approx_time(void)
Definition: approx_time.c:32
static int WRA_WAS_OUTDATED(was_router_added_t s)
Definition: routerlist.h:114
Header file for hs_cache.c.
int dir_split_resource_into_spoolable(const char *resource, dir_spool_source_t source, smartlist_t *spool_out, int *compressed_out, int flags)
Definition: directory.c:626
long tor_parse_long(const char *s, int base, long min, long max, int *ok, char **next)
Definition: parse_int.c:56
compression_level_t
Definition: compress.h:35
compress_method_t compression_method_get_by_name(const char *name)
Definition: compress.c:403
int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:506
#define CONN_TYPE_DIR
Definition: connection.h:35
uint8_t purpose
compress_method_t
Definition: compress.h:21
buf_t * buf_new_with_capacity(size_t size)
Definition: buffers.c:354
int crypto_digest256(char *digest, const char *m, size_t len, digest_algorithm_t algorithm)
int dirvote_add_signatures(const char *detached_signatures_body, const char *source, const char **msg)
Definition: dirvote.c:3634
was_router_added_t dirserv_add_multiple_descriptors(const char *desc, size_t desclen, uint8_t purpose, const char *source, const char **msg)
#define LD_PROTOCOL
Definition: log.h:69
int parse_http_command(const char *headers, char **command_out, char **url_out)
Definition: directory.c:175
int strcasecmpstart(const char *s1, const char *s2)
Definition: util_string.c:216
#define DIR_CONN_STATE_SERVER_WRITING
Definition: directory.h:29
Header file for networkstatus.c.
Header file for routerlist.c.
int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep, int flags, int max)