dircache.c File Reference

Cache directories and serve them to clients. More...

#include "core/or/or.h"
#include "app/config/config.h"
#include "app/config/resolve_addr.h"
#include "core/mainloop/connection.h"
#include "core/or/relay.h"
#include "feature/dirauth/dirvote.h"
#include "feature/dirauth/authmode.h"
#include "feature/dirauth/process_descs.h"
#include "feature/dircache/conscache.h"
#include "feature/dircache/consdiffmgr.h"
#include "feature/dircache/dircache.h"
#include "feature/dircache/dirserv.h"
#include "feature/dircommon/directory.h"
#include "feature/dircommon/fp_pair.h"
#include "feature/hs/hs_cache.h"
#include "feature/nodelist/authcert.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/nodelist/routerlist.h"
#include "feature/relay/relay_config.h"
#include "feature/relay/routermode.h"
#include "feature/stats/geoip_stats.h"
#include "feature/stats/rephist.h"
#include "lib/compress/compress.h"
#include "feature/dircache/cached_dir_st.h"
#include "feature/dircommon/dir_connection_st.h"
#include "feature/nodelist/authority_cert_st.h"
#include "feature/nodelist/networkstatus_st.h"
#include "feature/nodelist/routerinfo_st.h"

Go to the source code of this file.

Data Structures
struct	get_handler_args_t
struct	url_table_ent_t
struct	parsed_consensus_request_t

Macros
#define	DIRCACHE_PRIVATE
#define	MAX_DIR_UL_SIZE   ((1<<24)-1) /* 16MB-1 */
#define	FULL_DIR_CACHE_LIFETIME   (60*60)
#define	RUNNINGROUTERS_CACHE_LIFETIME   (20*60)
#define	DIRPORTFRONTPAGE_CACHE_LIFETIME   (20*60)
#define	NETWORKSTATUS_CACHE_LIFETIME   (5*60)
#define	ROUTERDESC_CACHE_LIFETIME   (30*60)
#define	ROUTERDESC_BY_DIGEST_CACHE_LIFETIME   (48*60*60)
#define	ROBOTS_CACHE_LIFETIME   (24*60*60)
#define	MICRODESC_CACHE_LIFETIME   (48*60*60)
#define	BANDWIDTH_CACHE_LIFETIME   (30*60)
#define	NOT_REASONABLY_LIVE_WARNING_INTERVAL   (60*60)
#define	FALLBACK_COMPRESS_METHOD   ZLIB_METHOD

Functions
STATIC int	parse_http_url (const char *headers, char **url)
static void	write_short_http_response (dir_connection_t *conn, int status, const char *reason_phrase)
static void	write_http_response_header_impl (dir_connection_t *conn, ssize_t length, const char *type, const char *encoding, const char *extra_headers, long cache_lifetime)
static void	write_http_response_headers (dir_connection_t *conn, ssize_t length, compress_method_t method, const char *extra_headers, long cache_lifetime)
static void	write_http_response_header (dir_connection_t *conn, ssize_t length, compress_method_t method, long cache_lifetime)
STATIC unsigned	parse_accept_encoding_header (const char *h)
static int	client_likes_consensus (const struct consensus_cache_entry_t *ent, const char *want_url)
STATIC compression_level_t	choose_compression_level (void)
static int	handle_get_frontpage (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_current_consensus (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_status_vote (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_microdesc (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_descriptor (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_keys (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_robots (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_networkstatus_bridges (dir_connection_t *conn, const get_handler_args_t *args)
static int	handle_get_next_bandwidth (dir_connection_t *conn, const get_handler_args_t *args)
STATIC int	directory_handle_command_get (dir_connection_t *conn, const char *headers, const char *req_body, size_t req_body_len)
static void	warn_consensus_is_not_reasonably_live (const struct consensus_cache_entry_t *consensus, const char *flavor, time_t now, bool is_too_new)
static int	parse_one_diff_hash (uint8_t *digest, const char *hex, const char *location, const char *action)
static int	parse_or_diff_from_header (smartlist_t **digests_out, const char *headers)
static struct consensus_cache_entry_t *	find_best_diff (const smartlist_t *digests, int flav, unsigned compression_methods, compress_method_t *compression_used_out)
static struct consensus_cache_entry_t *	find_best_consensus (int flav, unsigned compression_methods, compress_method_t *compression_used_out)
static compress_method_t	find_best_compression_method (unsigned compression_methods, int stream)
static int	digest_list_contains_best_consensus (consensus_flavor_t flavor, const smartlist_t *digests)
static void	parsed_consensus_request_clear (parsed_consensus_request_t *req)
static int	parse_consensus_request (parsed_consensus_request_t *out, const get_handler_args_t *args)
STATIC int	handle_get_hs_descriptor_v3 (dir_connection_t *conn, const get_handler_args_t *args)
STATIC int	parse_hs_version_from_post (const char *url, const char *prefix, const char **end_pos)
STATIC int	handle_post_hs_descriptor (const char *url, const char *body)
STATIC int	directory_handle_command_post (dir_connection_t *conn, const char *headers, const char *body, size_t body_len)
static void	http_set_address_origin (const char *headers, connection_t *conn)
int	directory_handle_command (dir_connection_t *conn)

Variables
static compress_method_t	srv_meth_pref_precompressed []
static compress_method_t	srv_meth_pref_streaming_compression []
static const url_table_ent_t	url_table []

Detailed Description

Cache directories and serve them to clients.

Definition in file dircache.c.

Macro Definition Documentation

FALLBACK_COMPRESS_METHOD

#define FALLBACK_COMPRESS_METHOD   ZLIB_METHOD

Fallback compression method. The fallback compression method is used in case a client requests a non-compressed document. We only store compressed documents, so we use this compression method to fetch the document and let the spooling system do the streaming decompression.

Definition at line 601 of file dircache.c.

FULL_DIR_CACHE_LIFETIME

#define FULL_DIR_CACHE_LIFETIME   (60*60)

HTTP cache control: how long do we tell proxies they can cache each kind of document we serve?

Definition at line 50 of file dircache.c.

MAX_DIR_UL_SIZE

#define MAX_DIR_UL_SIZE   ((1<<24)-1) /* 16MB-1 */

Maximum size, in bytes, for any directory object that we're accepting as an upload.

Definition at line 46 of file dircache.c.

Function Documentation

choose_compression_level()

STATIC compression_level_t choose_compression_level

(

void

)

Return the compression level we should use for sending a compressed response of size n_bytes.

Definition at line 298 of file dircache.c.

client_likes_consensus()

static int client_likes_consensus ( const struct consensus_cache_entry_t *  ent, const char *  want_url  )

static

Decide whether a client would accept the consensus we have.

Clients can say they only want a consensus if it's signed by more than half the authorities in a list. They pass this list in the url as "...consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>".

fpr may be an abbreviated fingerprint, i.e. only a left substring of the full authority identity digest. (Only strings of even length, i.e. encodings of full bytes, are handled correctly. In the case of an odd number of hex digits the last one is silently ignored.)

Returns 1 if more than half of the requested authorities signed the consensus, 0 otherwise.

Definition at line 258 of file dircache.c.

digest_list_contains_best_consensus()

static int digest_list_contains_best_consensus ( consensus_flavor_t  flavor, const smartlist_t *  digests  )

static

Check if any of the digests in digests matches the latest consensus flavor (given in flavor) that we have available.

Definition at line 711 of file dircache.c.

directory_handle_command()

int directory_handle_command

(

dir_connection_t *

conn

)

Called when a dirserver receives data on a directory connection; looks for an HTTP request. If the request is complete, remove it from the inbuf, try to process it; otherwise, leave it on the buffer. Return a 0 on success, or -1 on error.

Definition at line 1724 of file dircache.c.

Referenced by connection_dir_process_inbuf().

directory_handle_command_get()

STATIC int directory_handle_command_get	(	dir_connection_t *	conn,
		const char *	headers,
		const char *	req_body,
		size_t	req_body_len
	)

Helper function: called when a dirserver gets a complete HTTP GET request. Look for a request for a directory or for a rendezvous service descriptor. On finding one, write a response into conn->outbuf. If the request is unrecognized, send a 404. Return 0 if we handled this successfully, or -1 if we need to close the connection.

Definition at line 390 of file dircache.c.

directory_handle_command_post()

STATIC int directory_handle_command_post	(	dir_connection_t *	conn,
		const char *	headers,
		const char *	body,
		size_t	body_len
	)

Helper function: called when a dirserver gets a complete HTTP POST request. Look for an uploaded server descriptor or rendezvous service descriptor. On finding one, process it and write a response into conn->outbuf. If the request is unrecognized, send a

1. Always return 0.

Definition at line 1567 of file dircache.c.

find_best_compression_method()

static compress_method_t find_best_compression_method ( unsigned  compression_methods, int  stream  )

static

Try to find the best supported compression method possible from a given compression_methods. Return NO_METHOD if no mutually supported compression method could be found.

Definition at line 685 of file dircache.c.

Referenced by handle_get_current_consensus(), handle_get_descriptor(), handle_get_keys(), and handle_get_microdesc().

find_best_consensus()

static struct consensus_cache_entry_t* find_best_consensus ( int  flav, unsigned  compression_methods, compress_method_t *  compression_used_out  )

static

Lookup the cached consensus document by the flavor found in flav. The preferred set of compression methods should be listed in the compression_methods bitfield. The compression method chosen (if any) is stored in compression_used_out.

Definition at line 650 of file dircache.c.

find_best_diff()

static struct consensus_cache_entry_t* find_best_diff ( const smartlist_t *  digests, int  flav, unsigned  compression_methods, compress_method_t *  compression_used_out  )

static

Try to find the best consensus diff possible in order to serve a client request for a diff from one of the consensuses in digests to the current consensus of flavor flav. The client supports the compression methods listed in the compression_methods bitfield: place the method chosen (if any) into compression_used_out.

Definition at line 611 of file dircache.c.

handle_get_current_consensus()

static int handle_get_current_consensus ( dir_connection_t *  conn, const get_handler_args_t *  args  )

static

Helper function for GET /tor/status-vote/current/consensus

Definition at line 848 of file dircache.c.

handle_get_descriptor()

static int handle_get_descriptor ( dir_connection_t *  conn, const get_handler_args_t *  args  )

static

Helper function for GET /tor/{server,extra}/...

Definition at line 1160 of file dircache.c.

handle_get_frontpage()

static int handle_get_frontpage ( dir_connection_t *  conn, const get_handler_args_t *  args  )

static

Helper function for GET / or GET /tor/

Definition at line 479 of file dircache.c.

handle_get_hs_descriptor_v3()

STATIC int handle_get_hs_descriptor_v3	(	dir_connection_t *	conn,
		const get_handler_args_t *	args
	)

Helper function for GET /tor/hs/3/.... Only for version 3.

Definition at line 1352 of file dircache.c.

handle_get_keys()

static int handle_get_keys ( dir_connection_t *  conn, const get_handler_args_t *  args  )

static

Helper function for GET /tor/keys/...

Definition at line 1255 of file dircache.c.

handle_get_microdesc()

static int handle_get_microdesc ( dir_connection_t *  conn, const get_handler_args_t *  args  )

static

Helper function for GET /tor/micro/d/...

Definition at line 1106 of file dircache.c.

handle_get_networkstatus_bridges()

static int handle_get_networkstatus_bridges ( dir_connection_t *  conn, const get_handler_args_t *  args  )

static

Helper function for GET /tor/networkstatus-bridges

Definition at line 1391 of file dircache.c.

handle_get_next_bandwidth()

static int handle_get_next_bandwidth ( dir_connection_t *  conn, const get_handler_args_t *  args  )

static

Helper function for GET the bandwidth file used for the next vote

Definition at line 1431 of file dircache.c.

handle_get_robots()

static int handle_get_robots ( dir_connection_t *  conn, const get_handler_args_t *  args  )

static

Helper function for GET robots.txt or /tor/robots.txt

Definition at line 1464 of file dircache.c.

handle_get_status_vote()

static int handle_get_status_vote ( dir_connection_t *  conn, const get_handler_args_t *  args  )

static

Helper function for GET /tor/status-vote/{current,next}/...

Definition at line 1019 of file dircache.c.

http_set_address_origin()

static void http_set_address_origin ( const char *  headers, connection_t *  conn  )

static

If headers indicates that a proxy was involved, then rewrite conn->address to describe our best guess of the address that originated this HTTP request.

Definition at line 1696 of file dircache.c.

parse_accept_encoding_header()

STATIC unsigned parse_accept_encoding_header

(

const char *

h

)

Parse the compression methods listed in an Accept-Encoding header h, and convert them to a bitfield where compression method x is supported if and only if 1 << x is set in the bitfield.

Definition at line 222 of file dircache.c.

parse_consensus_request()

static int parse_consensus_request ( parsed_consensus_request_t *  out, const get_handler_args_t *  args  )

static

Parse the URL and relevant headers of args for a current-consensus request to learn what flavor of consensus we want, what keys it must be signed with, and what diffs we would accept (or demand) instead. Return 0 on success and -1 on failure.

Definition at line 773 of file dircache.c.

parse_http_url()

STATIC int parse_http_url	(	const char *	headers,
		char **	url
	)

Parse an HTTP request string headers of the form

* "\%s [http[s]://]\%s HTTP/1..."
* 

If it's well-formed, strdup the second %s into *url, and nul-terminate it. If the url doesn't start with "/tor/", rewrite it so it does. Return 0. Otherwise, return -1.

Definition at line 70 of file dircache.c.

parse_one_diff_hash()

static int parse_one_diff_hash ( uint8_t *  digest, const char *  hex, const char *  location, const char *  action  )

static

Parse a single hex-encoded sha3-256 digest from hex into digest. Return 0 on success. On failure, report that the hash came from location, report that we are taking action with it, and return -1.

Definition at line 554 of file dircache.c.

parse_or_diff_from_header()

static int parse_or_diff_from_header ( smartlist_t **  digests_out, const char *  headers  )

static

If there is an X-Or-Diff-From-Consensus header included in headers, set digest_out to a new smartlist containing every 256-bit hex-encoded digest listed in that header and return 0. Otherwise return -1.

Definition at line 573 of file dircache.c.

parsed_consensus_request_clear()

static void parsed_consensus_request_clear ( parsed_consensus_request_t *  req )

static

Remove all data held in req. Do not free req itself, since it is stack-allocated.

Definition at line 754 of file dircache.c.

warn_consensus_is_not_reasonably_live()

static void warn_consensus_is_not_reasonably_live ( const struct consensus_cache_entry_t *  consensus, const char *  flavor, time_t  now, bool  is_too_new  )

static

Warn that the cached consensus consensus of type flavor too new or too old, based on is_too_new, and will not be served to clients. Rate-limit the warning to avoid logging an entry on every request.

Definition at line 507 of file dircache.c.

write_http_response_header()

static void write_http_response_header ( dir_connection_t *  conn, ssize_t  length, compress_method_t  method, long  cache_lifetime  )

static

As write_http_response_headers, but assumes extra_headers is NULL

Definition at line 192 of file dircache.c.

Referenced by handle_get_hs_descriptor_v3(), and handle_get_robots().

write_http_response_header_impl()

static void write_http_response_header_impl ( dir_connection_t *  conn, ssize_t  length, const char *  type, const char *  encoding, const char *  extra_headers, long  cache_lifetime  )

static

Write the header for an HTTP/1.0 response onto conn->outbuf, with type as the Content-Type.

If length is nonnegative, it is the Content-Length. If encoding is provided, it is the Content-Encoding. If cache_lifetime is greater than 0, the content may be cached for up to cache_lifetime seconds. Otherwise, the content may not be cached.

Definition at line 127 of file dircache.c.

Referenced by handle_get_frontpage().

write_http_response_headers()

static void write_http_response_headers ( dir_connection_t *  conn, ssize_t  length, compress_method_t  method, const char *  extra_headers, long  cache_lifetime  )

static

As write_http_response_header_impl, but translates method into encoding

Definition at line 179 of file dircache.c.

write_short_http_response()

static void write_short_http_response ( dir_connection_t *  conn, int  status, const char *  reason_phrase  )

static

Create an http response for the client conn out of status and reason_phrase. Write it to conn.

Definition at line 92 of file dircache.c.

Referenced by handle_get_hs_descriptor_v3().

Variable Documentation

srv_meth_pref_precompressed

compress_method_t srv_meth_pref_precompressed[]

static

Initial value:

= {
  LZMA_METHOD,
  ZSTD_METHOD,
  ZLIB_METHOD,
  GZIP_METHOD,
  NO_METHOD
}

Array of compression methods to use (if supported) for serving precompressed data, ordered from best to worst.

Definition at line 201 of file dircache.c.

Referenced by find_best_compression_method(), find_best_consensus(), and find_best_diff().

srv_meth_pref_streaming_compression

compress_method_t srv_meth_pref_streaming_compression[]

static

Initial value:

= {
  ZSTD_METHOD,
  ZLIB_METHOD,
  GZIP_METHOD,
  NO_METHOD
}

Array of compression methods to use (if supported) for serving streamed data, ordered from best to worst.

Definition at line 211 of file dircache.c.

Referenced by find_best_compression_method().

url_table

const url_table_ent_t url_table[]

static

Initial value:

= {
  { "/tor/", 0, handle_get_frontpage },
  { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
  { "/tor/status-vote/current/", 1, handle_get_status_vote },
  { "/tor/status-vote/next/bandwidth", 0, handle_get_next_bandwidth },
  { "/tor/status-vote/next/", 1, handle_get_status_vote },
  { "/tor/micro/d/", 1, handle_get_microdesc },
  { "/tor/server/", 1, handle_get_descriptor },
  { "/tor/extra/", 1, handle_get_descriptor },
  { "/tor/keys/", 1, handle_get_keys },
  { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 },
  { "/tor/robots.txt", 0, handle_get_robots },
  { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges },
  { NULL, 0, NULL },
}

Table for handling GET requests.

Definition at line 366 of file dircache.c.