tor  0.4.0.1-alpha
dircache.c
1 /* Copyright (c) 2001-2004, Roger Dingledine.
2  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
3  * Copyright (c) 2007-2019, The Tor Project, Inc. */
4 /* See LICENSE for licensing information */
5 
6 #define DIRCACHE_PRIVATE
7 
8 #include "core/or/or.h"
9 
10 #include "app/config/config.h"
12 #include "core/or/relay.h"
16 #include "feature/dircache/conscache.h"
17 #include "feature/dircache/consdiffmgr.h"
22 #include "feature/hs/hs_cache.h"
27 #include "feature/rend/rendcache.h"
29 #include "feature/stats/rephist.h"
30 #include "lib/compress/compress.h"
31 
32 #include "feature/dircache/cached_dir_st.h"
33 #include "feature/dircommon/dir_connection_st.h"
34 #include "feature/nodelist/authority_cert_st.h"
35 #include "feature/nodelist/networkstatus_st.h"
36 #include "feature/nodelist/routerinfo_st.h"
37 
40 #define MAX_DIR_UL_SIZE ((1<<24)-1) /* 16MB-1 */
41 
44 #define FULL_DIR_CACHE_LIFETIME (60*60)
45 #define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
46 #define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60)
47 #define NETWORKSTATUS_CACHE_LIFETIME (5*60)
48 #define ROUTERDESC_CACHE_LIFETIME (30*60)
49 #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
50 #define ROBOTS_CACHE_LIFETIME (24*60*60)
51 #define MICRODESC_CACHE_LIFETIME (48*60*60)
52 
62 STATIC int
63 parse_http_url(const char *headers, char **url)
64 {
65  char *command = NULL;
66  if (parse_http_command(headers, &command, url) < 0) {
67  return -1;
68  }
69  if (strcmpstart(*url, "/tor/")) {
70  char *new_url = NULL;
71  tor_asprintf(&new_url, "/tor%s%s",
72  *url[0] == '/' ? "" : "/",
73  *url);
74  tor_free(*url);
75  *url = new_url;
76  }
77  tor_free(command);
78  return 0;
79 }
80 
84 static void
85 write_short_http_response(dir_connection_t *conn, int status,
86  const char *reason_phrase)
87 {
88  char *buf = NULL;
89  char *datestring = NULL;
90 
91  IF_BUG_ONCE(!reason_phrase) { /* bullet-proofing */
92  reason_phrase = "unspecified";
93  }
94 
95  if (server_mode(get_options())) {
96  /* include the Date: header, but only if we're a relay or bridge */
97  char datebuf[RFC1123_TIME_LEN+1];
98  format_rfc1123_time(datebuf, time(NULL));
99  tor_asprintf(&datestring, "Date: %s\r\n", datebuf);
100  }
101 
102  tor_asprintf(&buf, "HTTP/1.0 %d %s\r\n%s\r\n",
103  status, reason_phrase, datestring?datestring:"");
104 
105  log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase);
106  connection_buf_add(buf, strlen(buf), TO_CONN(conn));
107 
108  tor_free(datestring);
109  tor_free(buf);
110 }
111 
119 static void
120 write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
121  const char *type, const char *encoding,
122  const char *extra_headers,
123  long cache_lifetime)
124 {
125  char date[RFC1123_TIME_LEN+1];
126  time_t now = time(NULL);
127  buf_t *buf = buf_new_with_capacity(1024);
128 
129  tor_assert(conn);
130 
131  format_rfc1123_time(date, now);
132 
133  buf_add_printf(buf, "HTTP/1.0 200 OK\r\nDate: %s\r\n", date);
134  if (type) {
135  buf_add_printf(buf, "Content-Type: %s\r\n", type);
136  }
137  if (!is_local_addr(&conn->base_.addr)) {
138  /* Don't report the source address for a nearby/private connection.
139  * Otherwise we tend to mis-report in cases where incoming ports are
140  * being forwarded to a Tor server running behind the firewall. */
141  buf_add_printf(buf, X_ADDRESS_HEADER "%s\r\n", conn->base_.address);
142  }
143  if (encoding) {
144  buf_add_printf(buf, "Content-Encoding: %s\r\n", encoding);
145  }
146  if (length >= 0) {
147  buf_add_printf(buf, "Content-Length: %ld\r\n", (long)length);
148  }
149  if (cache_lifetime > 0) {
150  char expbuf[RFC1123_TIME_LEN+1];
151  format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime));
152  /* We could say 'Cache-control: max-age=%d' here if we start doing
153  * http/1.1 */
154  buf_add_printf(buf, "Expires: %s\r\n", expbuf);
155  } else if (cache_lifetime == 0) {
156  /* We could say 'Cache-control: no-cache' here if we start doing
157  * http/1.1 */
158  buf_add_string(buf, "Pragma: no-cache\r\n");
159  }
160  if (extra_headers) {
161  buf_add_string(buf, extra_headers);
162  }
163  buf_add_string(buf, "\r\n");
164 
165  connection_buf_add_buf(TO_CONN(conn), buf);
166  buf_free(buf);
167 }
168 
171 static void
172 write_http_response_headers(dir_connection_t *conn, ssize_t length,
173  compress_method_t method,
174  const char *extra_headers, long cache_lifetime)
175 {
176  write_http_response_header_impl(conn, length,
177  "text/plain",
179  extra_headers,
180  cache_lifetime);
181 }
182 
184 static void
185 write_http_response_header(dir_connection_t *conn, ssize_t length,
186  compress_method_t method,
187  long cache_lifetime)
188 {
189  write_http_response_headers(conn, length, method, NULL, cache_lifetime);
190 }
191 
194 static compress_method_t srv_meth_pref_precompressed[] = {
195  LZMA_METHOD,
196  ZSTD_METHOD,
197  ZLIB_METHOD,
198  GZIP_METHOD,
199  NO_METHOD
200 };
201 
204 static compress_method_t srv_meth_pref_streaming_compression[] = {
205  ZSTD_METHOD,
206  ZLIB_METHOD,
207  GZIP_METHOD,
208  NO_METHOD
209 };
210 
214 STATIC unsigned
215 parse_accept_encoding_header(const char *h)
216 {
217  unsigned result = (1u << NO_METHOD);
218  smartlist_t *methods = smartlist_new();
219  smartlist_split_string(methods, h, ",",
220  SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0);
221 
222  SMARTLIST_FOREACH_BEGIN(methods, const char *, m) {
224  if (method != UNKNOWN_METHOD) {
225  tor_assert(((unsigned)method) < 8*sizeof(unsigned));
226  result |= (1u << method);
227  }
228  } SMARTLIST_FOREACH_END(m);
229  SMARTLIST_FOREACH_BEGIN(methods, char *, m) {
230  tor_free(m);
231  } SMARTLIST_FOREACH_END(m);
232  smartlist_free(methods);
233  return result;
234 }
235 
250 static int
251 client_likes_consensus(const struct consensus_cache_entry_t *ent,
252  const char *want_url)
253 {
254  smartlist_t *voters = smartlist_new();
255  int need_at_least;
256  int have = 0;
257 
258  if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) {
259  smartlist_free(voters);
260  return 1; // We don't know the voters; assume the client won't mind. */
261  }
262 
263  smartlist_t *want_authorities = smartlist_new();
264  dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0);
265  need_at_least = smartlist_len(want_authorities)/2+1;
266 
267  SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) {
268 
269  SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) {
270  if (!strcasecmpstart(digest, want_digest)) {
271  have++;
272  break;
273  };
274  } SMARTLIST_FOREACH_END(digest);
275 
276  /* early exit, if we already have enough */
277  if (have >= need_at_least)
278  break;
279  } SMARTLIST_FOREACH_END(want_digest);
280 
281  SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d));
282  smartlist_free(want_authorities);
283  SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp));
284  smartlist_free(voters);
285  return (have >= need_at_least);
286 }
287 
290 STATIC compression_level_t
291 choose_compression_level(ssize_t n_bytes)
292 {
294  return HIGH_COMPRESSION; /* we have plenty of RAM. */
295  } else if (n_bytes < 0) {
296  return HIGH_COMPRESSION; /* unknown; might be big. */
297  } else if (n_bytes < 1024) {
298  return LOW_COMPRESSION;
299  } else if (n_bytes < 2048) {
300  return MEDIUM_COMPRESSION;
301  } else {
302  return HIGH_COMPRESSION;
303  }
304 }
305 
307 typedef struct get_handler_args_t {
315  const char *url;
317  const char *headers;
319 
330 typedef struct url_table_ent_s {
331  const char *string;
332  int is_prefix;
333  int (*handler)(dir_connection_t *conn, const get_handler_args_t *args);
335 
336 static int handle_get_frontpage(dir_connection_t *conn,
337  const get_handler_args_t *args);
338 static int handle_get_current_consensus(dir_connection_t *conn,
339  const get_handler_args_t *args);
340 static int handle_get_status_vote(dir_connection_t *conn,
341  const get_handler_args_t *args);
342 static int handle_get_microdesc(dir_connection_t *conn,
343  const get_handler_args_t *args);
344 static int handle_get_descriptor(dir_connection_t *conn,
345  const get_handler_args_t *args);
346 static int handle_get_keys(dir_connection_t *conn,
347  const get_handler_args_t *args);
348 static int handle_get_hs_descriptor_v2(dir_connection_t *conn,
349  const get_handler_args_t *args);
350 static int handle_get_robots(dir_connection_t *conn,
351  const get_handler_args_t *args);
352 static int handle_get_networkstatus_bridges(dir_connection_t *conn,
353  const get_handler_args_t *args);
354 
356 static const url_table_ent_t url_table[] = {
357  { "/tor/", 0, handle_get_frontpage },
358  { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
359  { "/tor/status-vote/current/", 1, handle_get_status_vote },
360  { "/tor/status-vote/next/", 1, handle_get_status_vote },
361  { "/tor/micro/d/", 1, handle_get_microdesc },
362  { "/tor/server/", 1, handle_get_descriptor },
363  { "/tor/extra/", 1, handle_get_descriptor },
364  { "/tor/keys/", 1, handle_get_keys },
365  { "/tor/rendezvous2/", 1, handle_get_hs_descriptor_v2 },
366  { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 },
367  { "/tor/robots.txt", 0, handle_get_robots },
368  { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges },
369  { NULL, 0, NULL },
370 };
371 
378 MOCK_IMPL(STATIC int,
379 directory_handle_command_get,(dir_connection_t *conn, const char *headers,
380  const char *req_body, size_t req_body_len))
381 {
382  char *url, *url_mem, *header;
383  time_t if_modified_since = 0;
384  int zlib_compressed_in_url;
385  unsigned compression_methods_supported;
386 
387  /* We ignore the body of a GET request. */
388  (void)req_body;
389  (void)req_body_len;
390 
391  log_debug(LD_DIRSERV,"Received GET command.");
392 
393  conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
394 
395  if (parse_http_url(headers, &url) < 0) {
396  write_short_http_response(conn, 400, "Bad request");
397  return 0;
398  }
399  if ((header = http_get_header(headers, "If-Modified-Since: "))) {
400  struct tm tm;
401  if (parse_http_time(header, &tm) == 0) {
402  if (tor_timegm(&tm, &if_modified_since)<0) {
403  if_modified_since = 0;
404  } else {
405  log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header));
406  }
407  }
408  /* The correct behavior on a malformed If-Modified-Since header is to
409  * act as if no If-Modified-Since header had been given. */
410  tor_free(header);
411  }
412  log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
413 
414  url_mem = url;
415  {
416  size_t url_len = strlen(url);
417 
418  zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z");
419  if (zlib_compressed_in_url) {
420  url[url_len-2] = '\0';
421  }
422  }
423 
424  if ((header = http_get_header(headers, "Accept-Encoding: "))) {
425  compression_methods_supported = parse_accept_encoding_header(header);
426  tor_free(header);
427  } else {
428  compression_methods_supported = (1u << NO_METHOD);
429  }
430  if (zlib_compressed_in_url) {
431  compression_methods_supported |= (1u << ZLIB_METHOD);
432  }
433 
434  /* Remove all methods that we don't both support. */
435  compression_methods_supported &= tor_compress_get_supported_method_bitmask();
436 
437  get_handler_args_t args;
438  args.url = url;
439  args.headers = headers;
440  args.if_modified_since = if_modified_since;
441  args.compression_supported = compression_methods_supported;
442 
443  int i, result = -1;
444  for (i = 0; url_table[i].string; ++i) {
445  int match;
446  if (url_table[i].is_prefix) {
447  match = !strcmpstart(url, url_table[i].string);
448  } else {
449  match = !strcmp(url, url_table[i].string);
450  }
451  if (match) {
452  result = url_table[i].handler(conn, &args);
453  goto done;
454  }
455  }
456 
457  /* we didn't recognize the url */
458  write_short_http_response(conn, 404, "Not found");
459  result = 0;
460 
461  done:
462  tor_free(url_mem);
463  return result;
464 }
465 
468 static int
469 handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
470 {
471  (void) args; /* unused */
472  const char *frontpage = get_dirportfrontpage();
473 
474  if (frontpage) {
475  size_t dlen;
476  dlen = strlen(frontpage);
477  /* Let's return a disclaimer page (users shouldn't use V1 anymore,
478  and caches don't fetch '/', so this is safe). */
479 
480  /* [We don't check for write_bucket_low here, since we want to serve
481  * this page no matter what.] */
482  write_http_response_header_impl(conn, dlen, "text/html", "identity",
483  NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME);
484  connection_buf_add(frontpage, dlen, TO_CONN(conn));
485  } else {
486  write_short_http_response(conn, 404, "Not found");
487  }
488  return 0;
489 }
490 
496 static void
497 warn_consensus_is_not_reasonably_live(
498  const struct consensus_cache_entry_t *consensus,
499  const char *flavor, time_t now, bool is_too_new)
500 {
501 #define NOT_REASONABLY_LIVE_WARNING_INTERVAL (60*60)
502  static ratelim_t warned[2] = { RATELIM_INIT(
503  NOT_REASONABLY_LIVE_WARNING_INTERVAL),
504  RATELIM_INIT(
505  NOT_REASONABLY_LIVE_WARNING_INTERVAL) };
506  char timestamp[ISO_TIME_LEN+1];
507  /* valid_after if is_too_new, valid_until if !is_too_new */
508  time_t valid_time = 0;
509  char *dupes = NULL;
510 
511  if (is_too_new) {
512  if (consensus_cache_entry_get_valid_after(consensus, &valid_time))
513  return;
514  dupes = rate_limit_log(&warned[1], now);
515  } else {
516  if (consensus_cache_entry_get_valid_until(consensus, &valid_time))
517  return;
518  dupes = rate_limit_log(&warned[0], now);
519  }
520 
521  if (dupes) {
522  format_local_iso_time(timestamp, valid_time);
523  log_warn(LD_DIRSERV, "Our %s%sconsensus is too %s, so we will not "
524  "serve it to clients. It was valid %s %s local time and we "
525  "continued to serve it for up to 24 hours %s.%s",
526  flavor ? flavor : "",
527  flavor ? " " : "",
528  is_too_new ? "new" : "old",
529  is_too_new ? "after" : "until",
530  timestamp,
531  is_too_new ? "before it was valid" : "after it expired",
532  dupes);
533  tor_free(dupes);
534  }
535 }
536 
543 static int
544 parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location,
545  const char *action)
546 {
547  if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) ==
548  DIGEST256_LEN) {
549  return 0;
550  } else {
551  log_fn(LOG_PROTOCOL_WARN, LD_DIR,
552  "%s contained bogus digest %s; %s.",
553  location, escaped(hex), action);
554  return -1;
555  }
556 }
557 
562 static int
563 parse_or_diff_from_header(smartlist_t **digests_out, const char *headers)
564 {
565  char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER);
566  if (hdr == NULL) {
567  return -1;
568  }
569  smartlist_t *hex_digests = smartlist_new();
570  *digests_out = smartlist_new();
571  smartlist_split_string(hex_digests, hdr, " ",
572  SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
573  SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) {
574  uint8_t digest[DIGEST256_LEN];
575  if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header",
576  "ignoring")) {
577  smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest)));
578  }
579  } SMARTLIST_FOREACH_END(hex);
580  SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp));
581  smartlist_free(hex_digests);
582  tor_free(hdr);
583  return 0;
584 }
585 
591 #define FALLBACK_COMPRESS_METHOD ZLIB_METHOD
592 
600 static struct consensus_cache_entry_t *
601 find_best_diff(const smartlist_t *digests, int flav,
602  unsigned compression_methods,
603  compress_method_t *compression_used_out)
604 {
605  struct consensus_cache_entry_t *result = NULL;
606 
607  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
608  unsigned u;
609  for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
610  compress_method_t method = srv_meth_pref_precompressed[u];
611  if (0 == (compression_methods & (1u<<method)))
612  continue; // client doesn't like this one, or we don't have it.
613  if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256,
614  diff_from, DIGEST256_LEN,
615  method) == CONSDIFF_AVAILABLE) {
616  tor_assert_nonfatal(result);
617  *compression_used_out = method;
618  return result;
619  }
620  }
621  } SMARTLIST_FOREACH_END(diff_from);
622 
623  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
624  if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from,
625  DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
626  tor_assert_nonfatal(result);
627  *compression_used_out = FALLBACK_COMPRESS_METHOD;
628  return result;
629  }
630  } SMARTLIST_FOREACH_END(diff_from);
631 
632  return NULL;
633 }
634 
639 static struct consensus_cache_entry_t *
640 find_best_consensus(int flav,
641  unsigned compression_methods,
642  compress_method_t *compression_used_out)
643 {
644  struct consensus_cache_entry_t *result = NULL;
645  unsigned u;
646 
647  for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
648  compress_method_t method = srv_meth_pref_precompressed[u];
649 
650  if (0 == (compression_methods & (1u<<method)))
651  continue;
652 
653  if (consdiffmgr_find_consensus(&result, flav,
654  method) == CONSDIFF_AVAILABLE) {
655  tor_assert_nonfatal(result);
656  *compression_used_out = method;
657  return result;
658  }
659  }
660 
661  if (consdiffmgr_find_consensus(&result, flav,
662  FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
663  tor_assert_nonfatal(result);
664  *compression_used_out = FALLBACK_COMPRESS_METHOD;
665  return result;
666  }
667 
668  return NULL;
669 }
670 
674 static compress_method_t
675 find_best_compression_method(unsigned compression_methods, int stream)
676 {
677  unsigned u;
678  compress_method_t *methods;
679  size_t length;
680 
681  if (stream) {
682  methods = srv_meth_pref_streaming_compression;
683  length = ARRAY_LENGTH(srv_meth_pref_streaming_compression);
684  } else {
685  methods = srv_meth_pref_precompressed;
686  length = ARRAY_LENGTH(srv_meth_pref_precompressed);
687  }
688 
689  for (u = 0; u < length; ++u) {
690  compress_method_t method = methods[u];
691  if (compression_methods & (1u<<method))
692  return method;
693  }
694 
695  return NO_METHOD;
696 }
697 
700 static int
701 digest_list_contains_best_consensus(consensus_flavor_t flavor,
702  const smartlist_t *digests)
703 {
704  const networkstatus_t *ns = NULL;
705 
706  if (digests == NULL)
707  return 0;
708 
709  ns = networkstatus_get_latest_consensus_by_flavor(flavor);
710 
711  if (ns == NULL)
712  return 0;
713 
714  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, digest) {
716  return 1;
717  } SMARTLIST_FOREACH_END(digest);
718 
719  return 0;
720 }
721 
724 typedef struct {
726  char *flavor;
732  const char *want_fps;
740 
743 static void
744 parsed_consensus_request_clear(parsed_consensus_request_t *req)
745 {
746  if (!req)
747  return;
748  tor_free(req->flavor);
749  if (req->diff_from_digests) {
750  SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d));
751  smartlist_free(req->diff_from_digests);
752  }
753  memset(req, 0, sizeof(parsed_consensus_request_t));
754 }
755 
762 static int
763 parse_consensus_request(parsed_consensus_request_t *out,
764  const get_handler_args_t *args)
765 {
766  const char *url = args->url;
767  memset(out, 0, sizeof(parsed_consensus_request_t));
768  out->flav = FLAV_NS;
769 
770  const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/";
771  const char CONSENSUS_FLAVORED_PREFIX[] =
772  "/tor/status-vote/current/consensus-";
773 
774  /* figure out the flavor if any, and who we wanted to sign the thing */
775  const char *after_flavor = NULL;
776 
777  if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
778  const char *f, *cp;
779  f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
780  cp = strchr(f, '/');
781  if (cp) {
782  after_flavor = cp+1;
783  out->flavor = tor_strndup(f, cp-f);
784  } else {
785  out->flavor = tor_strdup(f);
786  }
787  int flav = networkstatus_parse_flavor_name(out->flavor);
788  if (flav < 0)
789  flav = FLAV_NS;
790  out->flav = flav;
791  } else {
792  if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
793  after_flavor = url+strlen(CONSENSUS_URL_PREFIX);
794  }
795 
796  /* see whether we've been asked explicitly for a diff from an older
797  * consensus. (The user might also have said that a diff would be okay,
798  * via X-Or-Diff-From-Consensus */
799  const char DIFF_COMPONENT[] = "diff/";
800  char *diff_hash_in_url = NULL;
801  if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) {
802  after_flavor += strlen(DIFF_COMPONENT);
803  const char *cp = strchr(after_flavor, '/');
804  if (cp) {
805  diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor);
806  out->want_fps = cp+1;
807  } else {
808  diff_hash_in_url = tor_strdup(after_flavor);
809  out->want_fps = NULL;
810  }
811  } else {
812  out->want_fps = after_flavor;
813  }
814 
815  if (diff_hash_in_url) {
816  uint8_t diff_from[DIGEST256_LEN];
817  out->diff_from_digests = smartlist_new();
818  out->diff_only = 1;
819  int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL",
820  "rejecting");
821  tor_free(diff_hash_in_url);
822  if (ok) {
824  tor_memdup(diff_from, DIGEST256_LEN));
825  } else {
826  return -1;
827  }
828  } else {
829  parse_or_diff_from_header(&out->diff_from_digests, args->headers);
830  }
831 
832  return 0;
833 }
834 
837 static int
838 handle_get_current_consensus(dir_connection_t *conn,
839  const get_handler_args_t *args)
840 {
841  const compress_method_t compress_method =
842  find_best_compression_method(args->compression_supported, 0);
843  const time_t if_modified_since = args->if_modified_since;
844  int clear_spool = 0;
845 
846  /* v3 network status fetch. */
847  long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
848 
849  time_t now = time(NULL);
851 
852  if (parse_consensus_request(&req, args) < 0) {
853  write_short_http_response(conn, 404, "Couldn't parse request");
854  goto done;
855  }
856 
857  if (digest_list_contains_best_consensus(req.flav,
858  req.diff_from_digests)) {
859  write_short_http_response(conn, 304, "Not modified");
861  goto done;
862  }
863 
864  struct consensus_cache_entry_t *cached_consensus = NULL;
865 
866  compress_method_t compression_used = NO_METHOD;
867  if (req.diff_from_digests) {
868  cached_consensus = find_best_diff(req.diff_from_digests, req.flav,
869  args->compression_supported,
870  &compression_used);
871  }
872 
873  if (req.diff_only && !cached_consensus) {
874  write_short_http_response(conn, 404, "No such diff available");
876  goto done;
877  }
878 
879  if (! cached_consensus) {
880  cached_consensus = find_best_consensus(req.flav,
881  args->compression_supported,
882  &compression_used);
883  }
884 
885  time_t valid_after, fresh_until, valid_until;
886  int have_valid_after = 0, have_fresh_until = 0, have_valid_until = 0;
887  if (cached_consensus) {
888  have_valid_after =
889  !consensus_cache_entry_get_valid_after(cached_consensus, &valid_after);
890  have_fresh_until =
891  !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
892  have_valid_until =
893  !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
894  }
895 
896  if (cached_consensus && have_valid_after &&
898  write_short_http_response(conn, 404, "Consensus is too new");
899  warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now,
900  1);
902  goto done;
903  } else if (
904  cached_consensus && have_valid_until &&
906  write_short_http_response(conn, 404, "Consensus is too old");
907  warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now,
908  0);
910  goto done;
911  }
912 
913  if (cached_consensus && req.want_fps &&
914  !client_likes_consensus(cached_consensus, req.want_fps)) {
915  write_short_http_response(conn, 404, "Consensus not signed by sufficient "
916  "number of requested authorities");
918  goto done;
919  }
920 
921  conn->spool = smartlist_new();
922  clear_spool = 1;
923  {
924  spooled_resource_t *spooled;
925  if (cached_consensus) {
926  spooled = spooled_resource_new_from_cache_entry(cached_consensus);
927  smartlist_add(conn->spool, spooled);
928  }
929  }
930 
931  lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0;
932 
933  size_t size_guess = 0;
934  int n_expired = 0;
935  dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since,
936  compress_method != NO_METHOD,
937  &size_guess,
938  &n_expired);
939 
940  if (!smartlist_len(conn->spool) && !n_expired) {
941  write_short_http_response(conn, 404, "Not found");
943  goto done;
944  } else if (!smartlist_len(conn->spool)) {
945  write_short_http_response(conn, 304, "Not modified");
947  goto done;
948  }
949 
950  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
951  log_debug(LD_DIRSERV,
952  "Client asked for network status lists, but we've been "
953  "writing too many bytes lately. Sending 503 Dir busy.");
954  write_short_http_response(conn, 503, "Directory busy, try again later");
956  goto done;
957  }
958 
959  tor_addr_t addr;
960  if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) {
962  &addr, NULL,
963  time(NULL));
965  /* Note that a request for a network status has started, so that we
966  * can measure the download time later on. */
967  if (conn->dirreq_id)
968  geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED);
969  else
970  geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess,
971  DIRREQ_DIRECT);
972  }
973 
974  /* Use this header to tell caches that the response depends on the
975  * X-Or-Diff-From-Consensus header (or lack thereof). */
976  const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n";
977 
978  clear_spool = 0;
979 
980  // The compress_method might have been NO_METHOD, but we store the data
981  // compressed. Decompress them using `compression_used`. See fallback code in
982  // find_best_consensus() and find_best_diff().
983  write_http_response_headers(conn, -1,
984  compress_method == NO_METHOD ?
985  NO_METHOD : compression_used,
986  vary_header,
987  smartlist_len(conn->spool) == 1 ? lifetime : 0);
988 
989  if (compress_method == NO_METHOD && smartlist_len(conn->spool))
990  conn->compress_state = tor_compress_new(0, compression_used,
991  HIGH_COMPRESSION);
992 
993  /* Prime the connection with some data. */
994  const int initial_flush_result = connection_dirserv_flushed_some(conn);
995  tor_assert_nonfatal(initial_flush_result == 0);
996  goto done;
997 
998  done:
999  parsed_consensus_request_clear(&req);
1000  if (clear_spool) {
1001  dir_conn_clear_spool(conn);
1002  }
1003  return 0;
1004 }
1005 
1008 static int
1009 handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
1010 {
1011  const char *url = args->url;
1012  {
1013  ssize_t body_len = 0;
1014  ssize_t estimated_len = 0;
1015  int lifetime = 60; /* XXXX?? should actually use vote intervals. */
1016  /* This smartlist holds strings that we can compress on the fly. */
1017  smartlist_t *items = smartlist_new();
1018  /* This smartlist holds cached_dir_t objects that have a precompressed
1019  * deflated version. */
1020  smartlist_t *dir_items = smartlist_new();
1021  dirvote_dirreq_get_status_vote(url, items, dir_items);
1022  if (!smartlist_len(dir_items) && !smartlist_len(items)) {
1023  write_short_http_response(conn, 404, "Not found");
1024  goto vote_done;
1025  }
1026 
1027  /* We're sending items from at most one kind of source */
1028  tor_assert_nonfatal(smartlist_len(items) == 0 ||
1029  smartlist_len(dir_items) == 0);
1030 
1031  int streaming;
1032  unsigned mask;
1033  if (smartlist_len(items)) {
1034  /* We're taking strings and compressing them on the fly. */
1035  streaming = 1;
1036  mask = ~0u;
1037  } else {
1038  /* We're taking cached_dir_t objects. We only have them uncompressed
1039  * or deflated. */
1040  streaming = 0;
1041  mask = (1u<<NO_METHOD) | (1u<<ZLIB_METHOD);
1042  }
1043  const compress_method_t compress_method = find_best_compression_method(
1044  args->compression_supported&mask, streaming);
1045 
1046  SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
1047  body_len += compress_method != NO_METHOD ?
1048  d->dir_compressed_len : d->dir_len);
1049  estimated_len += body_len;
1050  SMARTLIST_FOREACH(items, const char *, item, {
1051  size_t ln = strlen(item);
1052  if (compress_method != NO_METHOD) {
1053  estimated_len += ln/2;
1054  } else {
1055  body_len += ln; estimated_len += ln;
1056  }
1057  });
1058 
1059  if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
1060  write_short_http_response(conn, 503, "Directory busy, try again later");
1061  goto vote_done;
1062  }
1063  write_http_response_header(conn, body_len ? body_len : -1,
1064  compress_method,
1065  lifetime);
1066 
1067  if (smartlist_len(items)) {
1068  if (compress_method != NO_METHOD) {
1069  conn->compress_state = tor_compress_new(1, compress_method,
1070  choose_compression_level(estimated_len));
1071  SMARTLIST_FOREACH(items, const char *, c,
1072  connection_buf_add_compress(c, strlen(c), conn, 0));
1073  connection_buf_add_compress("", 0, conn, 1);
1074  } else {
1075  SMARTLIST_FOREACH(items, const char *, c,
1076  connection_buf_add(c, strlen(c), TO_CONN(conn)));
1077  }
1078  } else {
1079  SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
1080  connection_buf_add(compress_method != NO_METHOD ?
1081  d->dir_compressed : d->dir,
1082  compress_method != NO_METHOD ?
1083  d->dir_compressed_len : d->dir_len,
1084  TO_CONN(conn)));
1085  }
1086  vote_done:
1087  smartlist_free(items);
1088  smartlist_free(dir_items);
1089  goto done;
1090  }
1091  done:
1092  return 0;
1093 }
1094 
1097 static int
1098 handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
1099 {
1100  const char *url = args->url;
1101  const compress_method_t compress_method =
1102  find_best_compression_method(args->compression_supported, 1);
1103  int clear_spool = 1;
1104  {
1105  conn->spool = smartlist_new();
1106 
1107  dir_split_resource_into_spoolable(url+strlen("/tor/micro/d/"),
1108  DIR_SPOOL_MICRODESC,
1109  conn->spool, NULL,
1110  DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ);
1111 
1112  size_t size_guess = 0;
1114  compress_method != NO_METHOD,
1115  &size_guess, NULL);
1116  if (smartlist_len(conn->spool) == 0) {
1117  write_short_http_response(conn, 404, "Not found");
1118  goto done;
1119  }
1120  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
1121  log_info(LD_DIRSERV,
1122  "Client asked for server descriptors, but we've been "
1123  "writing too many bytes lately. Sending 503 Dir busy.");
1124  write_short_http_response(conn, 503, "Directory busy, try again later");
1125  goto done;
1126  }
1127 
1128  clear_spool = 0;
1129  write_http_response_header(conn, -1,
1130  compress_method,
1131  MICRODESC_CACHE_LIFETIME);
1132 
1133  if (compress_method != NO_METHOD)
1134  conn->compress_state = tor_compress_new(1, compress_method,
1135  choose_compression_level(size_guess));
1136 
1137  const int initial_flush_result = connection_dirserv_flushed_some(conn);
1138  tor_assert_nonfatal(initial_flush_result == 0);
1139  goto done;
1140  }
1141 
1142  done:
1143  if (clear_spool) {
1144  dir_conn_clear_spool(conn);
1145  }
1146  return 0;
1147 }
1148 
1151 static int
1152 handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
1153 {
1154  const char *url = args->url;
1155  const compress_method_t compress_method =
1156  find_best_compression_method(args->compression_supported, 1);
1157  const or_options_t *options = get_options();
1158  int clear_spool = 1;
1159  if (!strcmpstart(url,"/tor/server/") ||
1160  (!options->BridgeAuthoritativeDir &&
1161  !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) {
1162  int res;
1163  const char *msg = NULL;
1164  int cache_lifetime = 0;
1165  int is_extra = !strcmpstart(url,"/tor/extra/");
1166  url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/");
1167  dir_spool_source_t source;
1168  time_t publish_cutoff = 0;
1169  if (!strcmpstart(url, "d/")) {
1170  source =
1171  is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST;
1172  } else {
1173  source =
1174  is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP;
1175  /* We only want to apply a publish cutoff when we're requesting
1176  * resources by fingerprint. */
1177  publish_cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
1178  }
1179 
1180  conn->spool = smartlist_new();
1181  res = dirserv_get_routerdesc_spool(conn->spool, url,
1182  source,
1184  &msg);
1185 
1186  if (!strcmpstart(url, "all")) {
1187  cache_lifetime = FULL_DIR_CACHE_LIFETIME;
1188  } else if (smartlist_len(conn->spool) == 1) {
1189  cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME;
1190  }
1191 
1192  size_t size_guess = 0;
1193  int n_expired = 0;
1194  dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff,
1195  compress_method != NO_METHOD,
1196  &size_guess, &n_expired);
1197 
1198  /* If we are the bridge authority and the descriptor is a bridge
1199  * descriptor, remember that we served this descriptor for desc stats. */
1200  /* XXXX it's a bit of a kludge to have this here. */
1201  if (get_options()->BridgeAuthoritativeDir &&
1202  source == DIR_SPOOL_SERVER_BY_FP) {
1203  SMARTLIST_FOREACH_BEGIN(conn->spool, spooled_resource_t *, spooled) {
1204  const routerinfo_t *router =
1205  router_get_by_id_digest((const char *)spooled->digest);
1206  /* router can be NULL here when the bridge auth is asked for its own
1207  * descriptor. */
1208  if (router && router->purpose == ROUTER_PURPOSE_BRIDGE)
1209  rep_hist_note_desc_served(router->cache_info.identity_digest);
1210  } SMARTLIST_FOREACH_END(spooled);
1211  }
1212 
1213  if (res < 0 || size_guess == 0 || smartlist_len(conn->spool) == 0) {
1214  if (msg == NULL)
1215  msg = "Not found";
1216  write_short_http_response(conn, 404, msg);
1217  } else {
1218  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
1219  log_info(LD_DIRSERV,
1220  "Client asked for server descriptors, but we've been "
1221  "writing too many bytes lately. Sending 503 Dir busy.");
1222  write_short_http_response(conn, 503,
1223  "Directory busy, try again later");
1224  dir_conn_clear_spool(conn);
1225  goto done;
1226  }
1227  write_http_response_header(conn, -1, compress_method, cache_lifetime);
1228  if (compress_method != NO_METHOD)
1229  conn->compress_state = tor_compress_new(1, compress_method,
1230  choose_compression_level(size_guess));
1231  clear_spool = 0;
1232  /* Prime the connection with some data. */
1233  int initial_flush_result = connection_dirserv_flushed_some(conn);
1234  tor_assert_nonfatal(initial_flush_result == 0);
1235  }
1236  goto done;
1237  }
1238  done:
1239  if (clear_spool)
1240  dir_conn_clear_spool(conn);
1241  return 0;
1242 }
1243 
1246 static int
1247 handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
1248 {
1249  const char *url = args->url;
1250  const compress_method_t compress_method =
1251  find_best_compression_method(args->compression_supported, 1);
1252  const time_t if_modified_since = args->if_modified_since;
1253  {
1254  smartlist_t *certs = smartlist_new();
1255  ssize_t len = -1;
1256  if (!strcmp(url, "/tor/keys/all")) {
1257  authority_cert_get_all(certs);
1258  } else if (!strcmp(url, "/tor/keys/authority")) {
1259  authority_cert_t *cert = get_my_v3_authority_cert();
1260  if (cert)
1261  smartlist_add(certs, cert);
1262  } else if (!strcmpstart(url, "/tor/keys/fp/")) {
1263  smartlist_t *fps = smartlist_new();
1264  dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"),
1265  fps, NULL,
1266  DSR_HEX|DSR_SORT_UNIQ);
1267  SMARTLIST_FOREACH(fps, char *, d, {
1269  if (c) smartlist_add(certs, c);
1270  tor_free(d);
1271  });
1272  smartlist_free(fps);
1273  } else if (!strcmpstart(url, "/tor/keys/sk/")) {
1274  smartlist_t *fps = smartlist_new();
1275  dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"),
1276  fps, NULL,
1277  DSR_HEX|DSR_SORT_UNIQ);
1278  SMARTLIST_FOREACH(fps, char *, d, {
1280  if (c) smartlist_add(certs, c);
1281  tor_free(d);
1282  });
1283  smartlist_free(fps);
1284  } else if (!strcmpstart(url, "/tor/keys/fp-sk/")) {
1285  smartlist_t *fp_sks = smartlist_new();
1286  dir_split_resource_into_fingerprint_pairs(url+strlen("/tor/keys/fp-sk/"),
1287  fp_sks);
1288  SMARTLIST_FOREACH(fp_sks, fp_pair_t *, pair, {
1290  pair->second);
1291  if (c) smartlist_add(certs, c);
1292  tor_free(pair);
1293  });
1294  smartlist_free(fp_sks);
1295  } else {
1296  write_short_http_response(conn, 400, "Bad request");
1297  goto keys_done;
1298  }
1299  if (!smartlist_len(certs)) {
1300  write_short_http_response(conn, 404, "Not found");
1301  goto keys_done;
1302  }
1303  SMARTLIST_FOREACH(certs, authority_cert_t *, c,
1304  if (c->cache_info.published_on < if_modified_since)
1305  SMARTLIST_DEL_CURRENT(certs, c));
1306  if (!smartlist_len(certs)) {
1307  write_short_http_response(conn, 304, "Not modified");
1308  goto keys_done;
1309  }
1310  len = 0;
1311  SMARTLIST_FOREACH(certs, authority_cert_t *, c,
1312  len += c->cache_info.signed_descriptor_len);
1313 
1314  if (global_write_bucket_low(TO_CONN(conn),
1315  compress_method != NO_METHOD ? len/2 : len,
1316  2)) {
1317  write_short_http_response(conn, 503, "Directory busy, try again later");
1318  goto keys_done;
1319  }
1320 
1321  write_http_response_header(conn,
1322  compress_method != NO_METHOD ? -1 : len,
1323  compress_method,
1324  60*60);
1325  if (compress_method != NO_METHOD) {
1326  conn->compress_state = tor_compress_new(1, compress_method,
1327  choose_compression_level(len));
1328  SMARTLIST_FOREACH(certs, authority_cert_t *, c,
1329  connection_buf_add_compress(
1330  c->cache_info.signed_descriptor_body,
1331  c->cache_info.signed_descriptor_len,
1332  conn, 0));
1333  connection_buf_add_compress("", 0, conn, 1);
1334  } else {
1335  SMARTLIST_FOREACH(certs, authority_cert_t *, c,
1336  connection_buf_add(c->cache_info.signed_descriptor_body,
1337  c->cache_info.signed_descriptor_len,
1338  TO_CONN(conn)));
1339  }
1340  keys_done:
1341  smartlist_free(certs);
1342  goto done;
1343  }
1344  done:
1345  return 0;
1346 }
1347 
1350 static int
1351 handle_get_hs_descriptor_v2(dir_connection_t *conn,
1352  const get_handler_args_t *args)
1353 {
1354  const char *url = args->url;
1355  if (connection_dir_is_encrypted(conn)) {
1356  /* Handle v2 rendezvous descriptor fetch request. */
1357  const char *descp;
1358  const char *query = url + strlen("/tor/rendezvous2/");
1359  if (rend_valid_descriptor_id(query)) {
1360  log_info(LD_REND, "Got a v2 rendezvous descriptor request for ID '%s'",
1361  safe_str(escaped(query)));
1362  switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) {
1363  case 1: /* valid */
1364  write_http_response_header(conn, strlen(descp), NO_METHOD, 0);
1365  connection_buf_add(descp, strlen(descp), TO_CONN(conn));
1366  break;
1367  case 0: /* well-formed but not present */
1368  write_short_http_response(conn, 404, "Not found");
1369  break;
1370  case -1: /* not well-formed */
1371  write_short_http_response(conn, 400, "Bad request");
1372  break;
1373  }
1374  } else { /* not well-formed */
1375  write_short_http_response(conn, 400, "Bad request");
1376  }
1377  goto done;
1378  } else {
1379  /* Not encrypted! */
1380  write_short_http_response(conn, 404, "Not found");
1381  }
1382  done:
1383  return 0;
1384 }
1385 
1388 STATIC int
1389 handle_get_hs_descriptor_v3(dir_connection_t *conn,
1390  const get_handler_args_t *args)
1391 {
1392  int retval;
1393  const char *desc_str = NULL;
1394  const char *pubkey_str = NULL;
1395  const char *url = args->url;
1396 
1397  /* Reject unencrypted dir connections */
1398  if (!connection_dir_is_encrypted(conn)) {
1399  write_short_http_response(conn, 404, "Not found");
1400  goto done;
1401  }
1402 
1403  /* After the path prefix follows the base64 encoded blinded pubkey which we
1404  * use to get the descriptor from the cache. Skip the prefix and get the
1405  * pubkey. */
1406  tor_assert(!strcmpstart(url, "/tor/hs/3/"));
1407  pubkey_str = url + strlen("/tor/hs/3/");
1408  retval = hs_cache_lookup_as_dir(HS_VERSION_THREE,
1409  pubkey_str, &desc_str);
1410  if (retval <= 0 || desc_str == NULL) {
1411  write_short_http_response(conn, 404, "Not found");
1412  goto done;
1413  }
1414 
1415  /* Found requested descriptor! Pass it to this nice client. */
1416  write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0);
1417  connection_buf_add(desc_str, strlen(desc_str), TO_CONN(conn));
1418 
1419  done:
1420  return 0;
1421 }
1422 
1425 static int
1426 handle_get_networkstatus_bridges(dir_connection_t *conn,
1427  const get_handler_args_t *args)
1428 {
1429  const char *headers = args->headers;
1430 
1431  const or_options_t *options = get_options();
1432  if (options->BridgeAuthoritativeDir &&
1433  options->BridgePassword_AuthDigest_ &&
1435  char *status;
1436  char digest[DIGEST256_LEN];
1437 
1438  char *header = http_get_header(headers, "Authorization: Basic ");
1439  if (header)
1440  crypto_digest256(digest, header, strlen(header), DIGEST_SHA256);
1441 
1442  /* now make sure the password is there and right */
1443  if (!header ||
1444  tor_memneq(digest,
1446  write_short_http_response(conn, 404, "Not found");
1447  tor_free(header);
1448  goto done;
1449  }
1450  tor_free(header);
1451 
1452  /* all happy now. send an answer. */
1453  status = networkstatus_getinfo_by_purpose("bridge", time(NULL));
1454  size_t dlen = strlen(status);
1455  write_http_response_header(conn, dlen, NO_METHOD, 0);
1456  connection_buf_add(status, dlen, TO_CONN(conn));
1457  tor_free(status);
1458  goto done;
1459  }
1460  done:
1461  return 0;
1462 }
1463 
1465 static int
1466 handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
1467 {
1468  (void)args;
1469  {
1470  const char robots[] = "User-agent: *\r\nDisallow: /\r\n";
1471  size_t len = strlen(robots);
1472  write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME);
1473  connection_buf_add(robots, len, TO_CONN(conn));
1474  }
1475  return 0;
1476 }
1477 
1478 /* Given the <b>url</b> from a POST request, try to extract the version number
1479  * using the provided <b>prefix</b>. The version should be after the prefix and
1480  * ending with the separator "/". For instance:
1481  * /tor/hs/3/publish
1482  *
1483  * On success, <b>end_pos</b> points to the position right after the version
1484  * was found. On error, it is set to NULL.
1485  *
1486  * Return version on success else negative value. */
1487 STATIC int
1488 parse_hs_version_from_post(const char *url, const char *prefix,
1489  const char **end_pos)
1490 {
1491  int ok;
1492  unsigned long version;
1493  const char *start;
1494  char *end = NULL;
1495 
1496  tor_assert(url);
1497  tor_assert(prefix);
1498  tor_assert(end_pos);
1499 
1500  /* Check if the prefix does start the url. */
1501  if (strcmpstart(url, prefix)) {
1502  goto err;
1503  }
1504  /* Move pointer to the end of the prefix string. */
1505  start = url + strlen(prefix);
1506  /* Try this to be the HS version and if we are still at the separator, next
1507  * will be move to the right value. */
1508  version = tor_parse_long(start, 10, 0, INT_MAX, &ok, &end);
1509  if (!ok) {
1510  goto err;
1511  }
1512 
1513  *end_pos = end;
1514  return (int) version;
1515  err:
1516  *end_pos = NULL;
1517  return -1;
1518 }
1519 
1520 /* Handle the POST request for a hidden service descripror. The request is in
1521  * <b>url</b>, the body of the request is in <b>body</b>. Return 200 on success
1522  * else return 400 indicating a bad request. */
1523 STATIC int
1524 handle_post_hs_descriptor(const char *url, const char *body)
1525 {
1526  int version;
1527  const char *end_pos;
1528 
1529  tor_assert(url);
1530  tor_assert(body);
1531 
1532  version = parse_hs_version_from_post(url, "/tor/hs/", &end_pos);
1533  if (version < 0) {
1534  goto err;
1535  }
1536 
1537  /* We have a valid version number, now make sure it's a publish request. Use
1538  * the end position just after the version and check for the command. */
1539  if (strcmpstart(end_pos, "/publish")) {
1540  goto err;
1541  }
1542 
1543  switch (version) {
1544  case HS_VERSION_THREE:
1545  if (hs_cache_store_as_dir(body) < 0) {
1546  goto err;
1547  }
1548  log_info(LD_REND, "Publish request for HS descriptor handled "
1549  "successfully.");
1550  break;
1551  default:
1552  /* Unsupported version, return a bad request. */
1553  goto err;
1554  }
1555 
1556  return 200;
1557  err:
1558  /* Bad request. */
1559  return 400;
1560 }
1561 
1567 MOCK_IMPL(STATIC int,
1568 directory_handle_command_post,(dir_connection_t *conn, const char *headers,
1569  const char *body, size_t body_len))
1570 {
1571  char *url = NULL;
1572  const or_options_t *options = get_options();
1573 
1574  log_debug(LD_DIRSERV,"Received POST command.");
1575 
1576  conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
1577 
1578  if (!public_server_mode(options)) {
1579  log_info(LD_DIR, "Rejected dir post request from %s "
1580  "since we're not a public relay.", conn->base_.address);
1581  write_short_http_response(conn, 503, "Not acting as a public relay");
1582  goto done;
1583  }
1584 
1585  if (parse_http_url(headers, &url) < 0) {
1586  write_short_http_response(conn, 400, "Bad request");
1587  return 0;
1588  }
1589  log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
1590 
1591  /* Handle v2 rendezvous service publish request. */
1592  if (connection_dir_is_encrypted(conn) &&
1593  !strcmpstart(url,"/tor/rendezvous2/publish")) {
1595  log_warn(LD_REND, "Rejected v2 rend descriptor (body size %d) from %s.",
1596  (int)body_len, conn->base_.address);
1597  write_short_http_response(conn, 400,
1598  "Invalid v2 service descriptor rejected");
1599  } else {
1600  write_short_http_response(conn, 200, "Service descriptor (v2) stored");
1601  log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted");
1602  }
1603  goto done;
1604  }
1605 
1606  /* Handle HS descriptor publish request. */
1607  /* XXX: This should be disabled with a consensus param until we want to
1608  * the prop224 be deployed and thus use. */
1609  if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) {
1610  const char *msg = "HS descriptor stored successfully.";
1611 
1612  /* We most probably have a publish request for an HS descriptor. */
1613  int code = handle_post_hs_descriptor(url, body);
1614  if (code != 200) {
1615  msg = "Invalid HS descriptor. Rejected.";
1616  }
1617  write_short_http_response(conn, code, msg);
1618  goto done;
1619  }
1620 
1621  if (!authdir_mode(options)) {
1622  /* we just provide cached directories; we don't want to
1623  * receive anything. */
1624  write_short_http_response(conn, 400, "Nonauthoritative directory does not "
1625  "accept posted server descriptors");
1626  goto done;
1627  }
1628 
1629  if (authdir_mode(options) &&
1630  !strcmp(url,"/tor/")) { /* server descriptor post */
1631  const char *msg = "[None]";
1632  uint8_t purpose = authdir_mode_bridge(options) ?
1633  ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
1635  purpose, conn->base_.address, &msg);
1636  tor_assert(msg);
1637 
1638  if (r == ROUTER_ADDED_SUCCESSFULLY) {
1639  write_short_http_response(conn, 200, msg);
1640  } else if (WRA_WAS_OUTDATED(r)) {
1641  write_http_response_header_impl(conn, -1, NULL, NULL,
1642  "X-Descriptor-Not-New: Yes\r\n", -1);
1643  } else {
1644  log_info(LD_DIRSERV,
1645  "Rejected router descriptor or extra-info from %s "
1646  "(\"%s\").",
1647  conn->base_.address, msg);
1648  write_short_http_response(conn, 400, msg);
1649  }
1650  goto done;
1651  }
1652 
1653  if (authdir_mode_v3(options) &&
1654  !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */
1655  const char *msg = "OK";
1656  int status;
1657  if (dirvote_add_vote(body, &msg, &status)) {
1658  write_short_http_response(conn, status, "Vote stored");
1659  } else {
1660  tor_assert(msg);
1661  log_warn(LD_DIRSERV, "Rejected vote from %s (\"%s\").",
1662  conn->base_.address, msg);
1663  write_short_http_response(conn, status, msg);
1664  }
1665  goto done;
1666  }
1667 
1668  if (authdir_mode_v3(options) &&
1669  !strcmp(url,"/tor/post/consensus-signature")) { /* sigs on consensus. */
1670  const char *msg = NULL;
1671  if (dirvote_add_signatures(body, conn->base_.address, &msg)>=0) {
1672  write_short_http_response(conn, 200, msg?msg:"Signatures stored");
1673  } else {
1674  log_warn(LD_DIR, "Unable to store signatures posted by %s: %s",
1675  conn->base_.address, msg?msg:"???");
1676  write_short_http_response(conn, 400,
1677  msg?msg:"Unable to store signatures");
1678  }
1679  goto done;
1680  }
1681 
1682  /* we didn't recognize the url */
1683  write_short_http_response(conn, 404, "Not found");
1684 
1685  done:
1686  tor_free(url);
1687  return 0;
1688 }
1689 
1693 static void
1694 http_set_address_origin(const char *headers, connection_t *conn)
1695 {
1696  char *fwd;
1697 
1698  fwd = http_get_header(headers, "Forwarded-For: ");
1699  if (!fwd)
1700  fwd = http_get_header(headers, "X-Forwarded-For: ");
1701  if (fwd) {
1702  tor_addr_t toraddr;
1703  if (tor_addr_parse(&toraddr,fwd) == -1 ||
1704  tor_addr_is_internal(&toraddr,0)) {
1705  log_debug(LD_DIR, "Ignoring local/internal IP %s", escaped(fwd));
1706  tor_free(fwd);
1707  return;
1708  }
1709 
1710  tor_free(conn->address);
1711  conn->address = tor_strdup(fwd);
1712  tor_free(fwd);
1713  }
1714 }
1715 
1721 int
1723 {
1724  char *headers=NULL, *body=NULL;
1725  size_t body_len=0;
1726  int r;
1727 
1728  tor_assert(conn);
1729  tor_assert(conn->base_.type == CONN_TYPE_DIR);
1730 
1732  &headers, MAX_HEADERS_SIZE,
1733  &body, &body_len, MAX_DIR_UL_SIZE, 0)) {
1734  case -1: /* overflow */
1735  log_warn(LD_DIRSERV,
1736  "Request too large from address '%s' to DirPort. Closing.",
1737  safe_str(conn->base_.address));
1738  return -1;
1739  case 0:
1740  log_debug(LD_DIRSERV,"command not all here yet.");
1741  return 0;
1742  /* case 1, fall through */
1743  }
1744 
1745  http_set_address_origin(headers, TO_CONN(conn));
1746  // we should escape headers here as well,
1747  // but we can't call escaped() twice, as it uses the same buffer
1748  //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, escaped(body));
1749 
1750  if (!strncasecmp(headers,"GET",3))
1751  r = directory_handle_command_get(conn, headers, body, body_len);
1752  else if (!strncasecmp(headers,"POST",4))
1753  r = directory_handle_command_post(conn, headers, body, body_len);
1754  else {
1755  log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
1756  "Got headers %s with unknown command. Closing.",
1757  escaped(headers));
1758  r = -1;
1759  }
1760 
1761  tor_free(headers); tor_free(body);
1762  return r;
1763 }
Header file for dirserv.c.
tor_compress_state_t * tor_compress_new(int compress, compress_method_t method, compression_level_t compression_level)
Definition: compress.c:481
void dir_conn_clear_spool(dir_connection_t *conn)
Definition: dirserv.c:905
void geoip_note_client_seen(geoip_client_action_t action, const tor_addr_t *addr, const char *transport_name, time_t now)
Definition: geoip_stats.c:227
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
#define TO_CONN(c)
Definition: or.h:735
int rend_valid_descriptor_id(const char *query)
Definition: rendcommon.c:732
was_router_added_t
Definition: routerlist.h:17
Header file for geoip_stats.c.
Header file for dircache.c.
tor_addr_t addr
int tor_timegm(const struct tm *tm, time_t *time_out)
Definition: time_fmt.c:92
void format_local_iso_time(char *buf, time_t t)
Definition: time_fmt.c:281
Header file for connection.c.
size_t dir_compressed_len
Definition: cached_dir_st.h:16
unsigned compression_supported
Definition: dircache.c:310
int directory_handle_command(dir_connection_t *conn)
Definition: dircache.c:1722
uint8_t state
Definition: connection_st.h:44
int authdir_mode(const or_options_t *options)
Definition: authmode.c:25
Headers for compress.c.
smartlist_t * diff_from_digests
Definition: dircache.c:735
char * networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
int rend_cache_lookup_v2_desc_as_dir(const char *desc_id, const char **desc)
Definition: rendcache.c:590
int dir_split_resource_into_fingerprint_pairs(const char *res, smartlist_t *pairs_out)
Definition: directory.c:485
Header file for directory.c.
void smartlist_add(smartlist_t *sl, void *element)
int parse_http_time(const char *date, struct tm *tm)
Definition: time_fmt.c:405
int dirserv_get_routerdesc_spool(smartlist_t *spool_out, const char *key, dir_spool_source_t source, int conn_is_encrypted, const char **msg_out)
Definition: dirserv.c:276
#define MAX_HEADERS_SIZE
Definition: or.h:123
char * rate_limit_log(ratelim_t *lim, time_t now)
Definition: ratelim.c:41
smartlist_t * spool
int authdir_mode_bridge(const or_options_t *options)
Definition: authmode.c:67
Header file for config.c.
Header file for authcert.c.
uint8_t digest[DIGEST256_LEN]
Definition: dirserv.h:55
uint8_t digest_sha3_as_signed[DIGEST256_LEN]
const char * headers
Definition: dircache.c:317
void rep_hist_note_desc_served(const char *desc)
Definition: rephist.c:2200
struct tor_compress_state_t * compress_state
unsigned tor_compress_get_supported_method_bitmask(void)
Definition: compress.c:328
int strcmpstart(const char *s1, const char *s2)
Definition: util_string.c:209
#define tor_free(p)
Definition: malloc.h:52
void authority_cert_get_all(smartlist_t *certs_out)
Definition: authcert.c:666
int crypto_digest256(char *digest, const char *m, size_t len, digest_algorithm_t algorithm)
int connection_fetch_from_buf_http(connection_t *conn, char **headers_out, size_t max_headerlen, char **body_out, size_t *body_used, size_t max_bodylen, int force_complete)
Definition: connection.c:3885
const uint8_t * body
Definition: conscache.c:51
int networkstatus_valid_until_is_reasonably_live(time_t valid_until, time_t now)
Definition: conscache.c:27
#define SMARTLIST_DEL_CURRENT(sl, var)
int connection_dirserv_flushed_some(dir_connection_t *conn)
Definition: dirserv.c:860
int global_write_bucket_low(connection_t *conn, size_t attempt, int priority)
Definition: connection.c:3191
pending_vote_t * dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
Definition: dirvote.c:3121
const char * url
Definition: dircache.c:315
signed_descriptor_t cache_info
spooled_resource_t * spooled_resource_new_from_cache_entry(consensus_cache_entry_t *entry)
Definition: dirserv.c:457
Header file for directory authority mode.
int networkstatus_parse_flavor_name(const char *flavname)
#define DIGEST256_LEN
Definition: digest_sizes.h:23
consensus_flavor_t flav
Definition: dircache.c:728
int rend_cache_store_v2_desc_as_dir(const char *desc)
Definition: rendcache.c:625
const char * compression_method_get_name(compress_method_t method)
Definition: compress.c:364
tor_assert(buffer)
int networkstatus_valid_after_is_reasonably_live(time_t valid_after, time_t now)
Header file for routermode.c.
int tor_memeq(const void *a, const void *b, size_t sz)
Definition: di_ops.c:107
int tor_asprintf(char **strp, const char *fmt,...)
Definition: printf.c:75
const routerinfo_t * router_get_by_id_digest(const char *digest)
Definition: routerlist.c:690
authority_cert_t * authority_cert_get_newest_by_id(const char *id_digest)
Definition: authcert.c:603
dir_spool_source_t
Definition: dirserv.h:20
Header file for rendcache.c.
Master header file for Tor-specific functionality.
#define LD_DIRSERV
Definition: log.h:86
int tor_addr_parse(tor_addr_t *addr, const char *src)
Definition: address.c:1185
void buf_add_string(buf_t *buf, const char *string)
Definition: buffers.c:559
Header file for rephist.c.
int have_been_under_memory_pressure(void)
Definition: relay.c:2662
unsigned int type
Definition: connection_st.h:45
time_t if_modified_since
Definition: dircache.c:313
void geoip_note_ns_response(geoip_ns_response_t response)
Definition: geoip_stats.c:393
authority_cert_t * authority_cert_get_by_sk_digest(const char *sk_digest)
Definition: authcert.c:623
#define LD_REND
Definition: log.h:80
int BridgeAuthoritativeDir
Header file for process_descs.c.
void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, dirreq_type_t type)
Definition: geoip_stats.c:529
void format_rfc1123_time(char *buf, time_t t)
Definition: time_fmt.c:178
#define LD_DIR
Definition: log.h:84
void connection_buf_add_buf(connection_t *conn, buf_t *buf)
Definition: connection.c:4357
char * http_get_header(const char *headers, const char *which)
Definition: directory.c:229
authority_cert_t * authority_cert_get_by_digests(const char *id_digest, const char *sk_digest)
Definition: authcert.c:650
char identity_digest[DIGEST_LEN]
Header file for fp_pair.c.
#define ROUTER_MAX_AGE_TO_PUBLISH
Definition: or.h:162
char * BridgePassword_AuthDigest_
#define IF_BUG_ONCE(cond)
Definition: util_bug.h:214
void buf_add_printf(buf_t *buf, const char *format,...)
Definition: buffers.c:566
Header file for dirvote.c.
Header file for relay.c.
#define SMARTLIST_FOREACH(sl, type, var, cmd)
int connection_dir_is_encrypted(const dir_connection_t *conn)
Definition: directory.c:160
const char * escaped(const char *s)
Definition: escape.c:126
void dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn, time_t cutoff, int compression, size_t *size_out, int *n_expired_out)
Definition: dirserv.c:762
consensus_flavor_t
Definition: or.h:863
#define ARRAY_LENGTH(x)
#define log_fn(severity, domain, args,...)
Definition: log.h:255
int dir_split_resource_into_fingerprints(const char *resource, smartlist_t *fp_out, int *compressed_out, int flags)
Definition: directory.c:544
static int WRA_WAS_OUTDATED(was_router_added_t s)
Definition: routerlist.h:111
Header file for hs_cache.c.
int dir_split_resource_into_spoolable(const char *resource, dir_spool_source_t source, smartlist_t *spool_out, int *compressed_out, int flags)
Definition: directory.c:626
long tor_parse_long(const char *s, int base, long min, long max, int *ok, char **next)
Definition: parse_int.c:56
compression_level_t
Definition: compress.h:35
compress_method_t compression_method_get_by_name(const char *name)
Definition: compress.c:403
int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:504
#define CONN_TYPE_DIR
Definition: connection.h:35
uint8_t purpose
compress_method_t
Definition: compress.h:21
buf_t * buf_new_with_capacity(size_t size)
Definition: buffers.c:354
int dirvote_add_signatures(const char *detached_signatures_body, const char *source, const char **msg)
Definition: dirvote.c:3624
was_router_added_t dirserv_add_multiple_descriptors(const char *desc, size_t desclen, uint8_t purpose, const char *source, const char **msg)
#define LD_PROTOCOL
Definition: log.h:68
int parse_http_command(const char *headers, char **command_out, char **url_out)
Definition: directory.c:175
int strcasecmpstart(const char *s1, const char *s2)
Definition: util_string.c:219
#define DIR_CONN_STATE_SERVER_WRITING
Definition: directory.h:29
Header file for networkstatus.c.
const char * safe_str(const char *address)
Definition: config.c:1076
Header file for routerlist.c.
int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep, int flags, int max)