Basic history and performance-tracking functionality. More...

#include "core/or/or.h"
#include "app/config/config.h"
#include "core/or/circuitlist.h"
#include "core/or/connection_or.h"
#include "feature/dirauth/authmode.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/nodelist/nodelist.h"
#include "feature/stats/predict_ports.h"
#include "feature/stats/connstats.h"
#include "feature/stats/rephist.h"
#include "lib/container/order.h"
#include "lib/crypt_ops/crypto_rand.h"
#include "lib/math/laplace.h"
#include "feature/nodelist/networkstatus_st.h"
#include "core/or/or_circuit_st.h"
#include <event2/dns.h>

Data Structures
struct	or_history_t

struct	padding_counts_t

struct	overload_stats_t

struct	dns_stats_t

struct	circ_buffer_stats_t

Macros
#define	REPHIST_PRIVATE

#define	STABILITY_EPSILON 0.0001

#define	STABILITY_ALPHA 0.95

#define	STABILITY_INTERVAL (126060)

#define	OVERLOAD_STATS_VERSION 1

#define	SET_TO_START_OF_HOUR(a)

#define	SUBTRACT_CLAMPED(var, penalty) do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0)

#define	PUT(s) STMT_BEGIN if (fputs((s),f)<0) goto err; STMT_END

#define	PRINTF(args) STMT_BEGIN if (fprintf args <0) goto err; STMT_END

#define	EXIT_STATS_ROUND_UP_BYTES 1024

#define	EXIT_STATS_ROUND_UP_STREAMS 4

#define	EXIT_STATS_NUM_PORTS 65536

#define	EXIT_STATS_TOP_N_PORTS 10

#define	SHARES 10

#define	hs_v2_stats_free(val) FREE_AND_NULL(hs_v2_stats_t, hs_v2_stats_free_, (val))

#define	hs_v3_stats_free(val) FREE_AND_NULL(hs_v3_stats_t, hs_v3_stats_free_, (val))

#define	REND_CELLS_DELTA_F 2048

#define	REND_CELLS_EPSILON 0.3

#define	REND_CELLS_BIN_SIZE 1024

#define	ONIONS_SEEN_DELTA_F 8

#define	ONIONS_SEEN_EPSILON 0.3

#define	ONIONS_SEEN_BIN_SIZE 8

#define	MIN_CELL_COUNTS_TO_PUBLISH 1

#define	ROUND_CELL_COUNTS_TO 10000

#define	ROUND_AND_SET_COUNT(x)

Functions
static dns_stats_t *	get_dns_stats_by_type (const int type)

uint64_t	rep_hist_get_n_dns_error (int type, uint8_t error)

uint64_t	rep_hist_get_n_dns_request (int type)

void	rep_hist_note_dns_error (int type, uint8_t error)

void	rep_hist_note_dns_request (int type)

static bool	overload_happened_recently (time_t overload_time, int n_hours)

uint64_t	rep_hist_get_n_read_limit_reached (void)

uint64_t	rep_hist_get_n_write_limit_reached (void)

char *	rep_hist_get_overload_general_line (void)

char *	rep_hist_get_overload_stats_lines (void)

void	rep_hist_note_overload (overload_type_t overload)

void	rep_hist_note_tcp_exhaustion (void)

uint64_t	rep_hist_get_n_tcp_exhaustion (void)

static or_history_t *	get_or_history (const char *id)

static void	free_or_history (void *_hist)

void	rep_hist_init (void)

void	rep_hist_note_router_reachable (const char id, const tor_addr_t at_addr, const uint16_t at_port, time_t when)

void	rep_hist_note_router_unreachable (const char *id, time_t when)

void	rep_hist_make_router_pessimal (const char *id, time_t when)

time_t	rep_hist_downrate_old_runs (time_t now)

static double	get_stability (or_history_t *hist, time_t when)

static long	get_total_weighted_time (or_history_t *hist, time_t when)

static double	get_weighted_fractional_uptime (or_history_t *hist, time_t when)

long	rep_hist_get_uptime (const char *id, time_t when)

double	rep_hist_get_stability (const char *id, time_t when)

double	rep_hist_get_weighted_fractional_uptime (const char *id, time_t when)

long	rep_hist_get_weighted_time_known (const char *id, time_t when)

int	rep_hist_have_measured_enough_stability (void)

void	rep_hist_dump_stats (time_t now, int severity)

void	rep_history_clean (time_t before)

int	rep_hist_record_mtbf_data (time_t now, int missing_means_down)

static int	find_next_with (smartlist_t sl, int i, const char prefix)

static int	parse_possibly_bad_iso_time (const char s, time_t time_out)

static time_t	correct_time (time_t t, time_t now, time_t stored_at, time_t started_measuring)

int	rep_hist_load_mtbf_data (time_t now)

void	rep_hist_exit_stats_init (time_t now)

void	rep_hist_reset_exit_stats (time_t now)

void	rep_hist_exit_stats_term (void)

static int	compare_int_ (const void x, const void y)

char *	rep_hist_format_exit_stats (time_t now)

time_t	rep_hist_exit_stats_write (time_t now)

void	rep_hist_note_exit_bytes (uint16_t port, size_t num_written, size_t num_read)

void	rep_hist_note_exit_stream_opened (uint16_t port)

void	rep_hist_buffer_stats_init (time_t now)

void	rep_hist_add_buffer_stats (double mean_num_cells_in_queue, double mean_time_cells_in_queue, uint32_t processed_cells)

void	rep_hist_buffer_stats_add_circ (circuit_t *circ, time_t end_of_interval)

static int	buffer_stats_compare_entries_ (const void _a, const void _b)

void	rep_hist_buffer_stats_term (void)

void	rep_hist_reset_buffer_stats (time_t now)

char *	rep_hist_format_buffer_stats (time_t now)

time_t	rep_hist_buffer_stats_write (time_t now)

void	rep_hist_desc_stats_init (time_t now)

static void	rep_hist_reset_desc_stats (time_t now)

void	rep_hist_desc_stats_term (void)

static char *	rep_hist_format_desc_stats (time_t now)

time_t	rep_hist_desc_stats_write (time_t now)

void	rep_hist_note_desc_served (const char *desc)

void	rep_hist_note_circuit_handshake_requested (uint16_t type)

void	rep_hist_note_circuit_handshake_assigned (uint16_t type)

void	rep_hist_note_circuit_handshake_dropped (uint16_t type)

int	rep_hist_get_circuit_handshake_requested (uint16_t type)

int	rep_hist_get_circuit_handshake_assigned (uint16_t type)

uint64_t	rep_hist_get_circuit_handshake_dropped (uint16_t type)

void	rep_hist_log_circuit_handshake_stats (time_t now)

static hs_v2_stats_t *	hs_v2_stats_new (void)

static void	hs_v2_stats_free_ (hs_v2_stats_t *victim_hs_v2_stats)

static void	rep_hist_reset_hs_v2_stats (time_t now)

static hs_v3_stats_t *	hs_v3_stats_new (void)

static void	hs_v3_stats_free_ (hs_v3_stats_t *victim_hs_v3_stats)

static void	rep_hist_reset_hs_v3_stats (time_t now)

STATIC bool	should_collect_v3_stats (void)

void	rep_hist_hsdir_stored_maybe_new_v3_onion (const uint8_t *blinded_key)

void	rep_hist_seen_new_rp_cell (bool is_v2)

void	rep_hist_hs_stats_init (time_t now)

void	rep_hist_hs_stats_term (void)

STATIC char *	rep_hist_format_hs_stats (time_t now, bool is_v3)

time_t	rep_hist_hs_stats_write (time_t now, bool is_v3)

void	rep_hist_note_negotiated_link_proto (unsigned link_proto, int started_here)

void	rep_hist_padding_count_timers (uint64_t num_timers)

void	rep_hist_padding_count_write (padding_type_t type)

void	rep_hist_padding_count_read (padding_type_t type)

void	rep_hist_reset_padding_counts (void)

void	rep_hist_prep_published_padding_counts (time_t now)

char *	rep_hist_get_padding_count_lines (void)

void	rep_hist_log_link_protocol_counts (void)

void	rep_hist_free_all (void)

Variables
uint64_t	rephist_total_alloc =0

uint32_t	rephist_total_num =0

static padding_counts_t	padding_current

static padding_counts_t	padding_published

static time_t	stability_last_downrated = 0

static time_t	started_tracking_stability = 0

static digestmap_t *	history_map = NULL

static overload_stats_t	overload_stats

static uint64_t	stats_n_read_limit_reached = 0

static uint64_t	stats_n_write_limit_reached = 0

static uint64_t	stats_n_tcp_exhaustion = 0

static dns_stats_t	dns_A_stats

static dns_stats_t	dns_PTR_stats

static dns_stats_t	dns_AAAA_stats

static int	n_bogus_times = 0

static uint64_t *	exit_bytes_read = NULL

static uint64_t *	exit_bytes_written = NULL

static uint32_t *	exit_streams = NULL

static time_t	start_of_exit_stats_interval

static time_t	start_of_buffer_stats_interval

static smartlist_t *	circuits_for_buffer_stats = NULL

static digestmap_t *	served_descs = NULL

static unsigned long	total_descriptor_downloads

static time_t	start_of_served_descs_stats_interval

static time_t	start_of_hs_v2_stats_interval

static hs_v2_stats_t *	hs_v2_stats = NULL

static time_t	start_of_hs_v3_stats_interval

static hs_v3_stats_t *	hs_v3_stats = NULL

static uint64_t	link_proto_count [MAX_LINK_PROTO+1][2]


STATIC int	onion_handshakes_requested [MAX_ONION_HANDSHAKE_TYPE+1] = {0}

STATIC int	onion_handshakes_assigned [MAX_ONION_HANDSHAKE_TYPE+1] = {0}

STATIC uint64_t	onion_handshakes_dropped [MAX_ONION_HANDSHAKE_TYPE+1] = {0}

Detailed Description

Basic history and performance-tracking functionality.

Basic history and performance-tracking functionality to remember which servers have worked in the past, how much bandwidth we've been using, which ports we tend to want, and so on; further, exit port statistics, cell statistics, and connection statistics.

The history and information tracked in this module could sensibly be divided into several categories:

Statistics used by authorities to remember the uptime and stability information about various relays, including "uptime", "weighted fractional uptime" and "mean time between failures".
Predicted ports, used by clients to remember how long it's been since they opened an exit connection to each given target port. Clients use this information in order to try to keep circuits open to exit nodes that can connect to the ports that they care about. (The predicted ports mechanism also handles predicted circuit usage that isn't port-specific, such as resolves, internal circuits, and so on.)
Public key operation counters, for tracking how many times we've done each public key operation. (This is unmaintained and we should remove it.)
Exit statistics by port, used by exits to keep track of the number of streams and bytes they've served at each exit port, so they can generate their exit-kibibytes-{read,written} and exit-streams-opened statistics.
Circuit stats, used by relays instances to tract circuit queue fullness and delay over time, and generate cell-processed-cells, cell-queued-cells, cell-time-in-queue, and cell-circuits-per-decile statistics.
Descriptor serving statistics, used by directory caches to track how many descriptors they've served.
Onion handshake statistics, used by relays to count how many TAP and ntor handshakes they've handled.
Hidden service statistics, used by relays to count rendezvous traffic and HSDir-stored descriptors.
Link protocol statistics, used by relays to count how many times each link protocol has been used.

The entry points for this module are scattered throughout the codebase. Sending data, receiving data, connecting to a relay, losing a connection to a relay, and so on can all trigger a change in our current stats. Relays also invoke this module in order to extract their statistics when building routerinfo and extrainfo objects in router.c.

TODO: This module should be broken up.

(The "rephist" name originally stood for "reputation and history". )

Definition in file rephist.c.

Macro Definition Documentation

◆ EXIT_STATS_NUM_PORTS

#define EXIT_STATS_NUM_PORTS 65536

Number of TCP ports

Definition at line 1320 of file rephist.c.

◆ EXIT_STATS_ROUND_UP_BYTES

#define EXIT_STATS_ROUND_UP_BYTES 1024

To what multiple should byte numbers be rounded up?

Definition at line 1316 of file rephist.c.

◆ EXIT_STATS_ROUND_UP_STREAMS

#define EXIT_STATS_ROUND_UP_STREAMS 4

To what multiple should stream counts be rounded up?

Definition at line 1318 of file rephist.c.

◆ EXIT_STATS_TOP_N_PORTS

#define EXIT_STATS_TOP_N_PORTS 10

Top n ports that will be included in exit stats.

Definition at line 1322 of file rephist.c.

◆ MIN_CELL_COUNTS_TO_PUBLISH

#define MIN_CELL_COUNTS_TO_PUBLISH 1

Copy our current cell counts into a structure for listing in our extra-info descriptor. Also perform appropriate rounding and redaction.

This function is called once every 24 hours.

Definition at line 2506 of file rephist.c.

◆ REND_CELLS_DELTA_F

#define REND_CELLS_DELTA_F 2048

Stats reporting code

Definition at line 2293 of file rephist.c.

◆ ROUND_AND_SET_COUNT

#define ROUND_AND_SET_COUNT ( x )

Value:

(x) = round_uint64_to_next_multiple_of((x), \

ROUND_CELL_COUNTS_TO)

round_uint64_to_next_multiple_of

uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor)

Definition: muldiv.c:50

◆ SET_TO_START_OF_HOUR

#define SET_TO_START_OF_HOUR ( a )

Value:

  STMT_BEGIN \
  (a) = approx_time() - (approx_time() % 3600); \
STMT_END

Round down the time in a to the beginning of the current hour

Definition at line 479 of file rephist.c.

◆ STABILITY_ALPHA

#define STABILITY_ALPHA 0.95

Value by which to discount all old intervals for MTBF purposes. This is compounded every STABILITY_INTERVAL.

Definition at line 103 of file rephist.c.

◆ STABILITY_EPSILON

#define STABILITY_EPSILON 0.0001

If the total weighted run count of all runs for a router ever falls below this amount, the router can be treated as having 0 MTBF.

Definition at line 100 of file rephist.c.

◆ STABILITY_INTERVAL

#define STABILITY_INTERVAL (12*60*60)

Interval at which to discount all old intervals for MTBF purposes.

Definition at line 105 of file rephist.c.

Function Documentation

◆ buffer_stats_compare_entries_()

static int buffer_stats_compare_entries_	(	const void **	_a,
		const void **	_b
	)

static

Sorting helper: return -1, 1, or 0 based on comparison of two circ_buffer_stats_t

Definition at line 1675 of file rephist.c.

◆ compare_int_()

static int compare_int_	(	const void *	x,
		const void *	y
	)

static

Helper for qsort: compare two ints. Does not handle overflow properly, but works fine for sorting an array of port numbers, which is what we use it for.

Definition at line 1373 of file rephist.c.

◆ correct_time()

static time_t correct_time	(	time_t	t,
		time_t	now,
		time_t	stored_at,
		time_t	started_measuring
	)

inlinestatic

We've read a time t from a file stored at stored_at, which says we started measuring at started_measuring. Return a new number that's about as much before now as t was before stored_at.

Definition at line 1105 of file rephist.c.

◆ find_next_with()

static int find_next_with	(	smartlist_t *	sl,
		int	i,
		const char *	prefix
	)

static

Helper: return the first j >= i such that !strcmpstart(sl[j], prefix) and such that no line sl[k] with i <= k < j starts with "R ". Return -1 if no such line exists.

Definition at line 1067 of file rephist.c.

◆ free_or_history()

static void free_or_history ( void * _hist )

static

Helper: free storage held by a single OR history entry.

Definition at line 559 of file rephist.c.

Referenced by rep_history_clean().

◆ get_dns_stats_by_type()

static dns_stats_t* get_dns_stats_by_type ( const int type )

inlinestatic

From a libevent record type, return a pointer to the corresponding DNS statistics store. NULL is returned if the type is unhandled.

Definition at line 255 of file rephist.c.

◆ get_or_history()

static or_history_t* get_or_history ( const char * id )

static

Return the or_history_t for the OR with identity digest id, creating it if necessary.

Definition at line 538 of file rephist.c.

Referenced by rep_hist_get_stability(), rep_hist_get_uptime(), rep_hist_get_weighted_fractional_uptime(), rep_hist_get_weighted_time_known(), rep_hist_make_router_pessimal(), rep_hist_note_router_reachable(), and rep_hist_note_router_unreachable().

◆ get_stability()

static double get_stability	(	or_history_t *	hist,
		time_t	when
	)

static

Helper: Return the weighted MTBF of the router with history hist.

Definition at line 763 of file rephist.c.

Referenced by rep_hist_dump_stats(), and rep_hist_get_stability().

◆ get_total_weighted_time()

static long get_total_weighted_time	(	or_history_t *	hist,
		time_t	when
	)

static

Return the total amount of time we've been observing, with each run of time downrated by the appropriate factor.

Definition at line 785 of file rephist.c.

Referenced by rep_hist_get_weighted_time_known().

◆ get_weighted_fractional_uptime()

static double get_weighted_fractional_uptime	(	or_history_t *	hist,
		time_t	when
	)

static

Helper: Return the weighted percent-of-time-online of the router with history hist.

Definition at line 799 of file rephist.c.

Referenced by rep_hist_get_weighted_fractional_uptime().

◆ hs_v2_stats_free_()

static void hs_v2_stats_free_ ( hs_v2_stats_t * victim_hs_v2_stats )

static

Free an hs_v2_stats_t structure.

Definition at line 2103 of file rephist.c.

◆ hs_v2_stats_new()

static hs_v2_stats_t* hs_v2_stats_new ( void )

static

HSv2 stats Allocate, initialize and return an hs_v2_stats_t structure.

Definition at line 2091 of file rephist.c.

Referenced by rep_hist_hs_stats_init(), and rep_hist_reset_hs_v2_stats().

◆ hs_v3_stats_free_()

static void hs_v3_stats_free_ ( hs_v3_stats_t * victim_hs_v3_stats )

static

Free an hs_v3_stats_t structure.

Definition at line 2176 of file rephist.c.

◆ hs_v3_stats_new()

static hs_v3_stats_t* hs_v3_stats_new ( void )

static

Allocate, initialize and return an hs_v3_stats_t structure.

Definition at line 2163 of file rephist.c.

Referenced by rep_hist_hs_stats_init(), and rep_hist_reset_hs_v3_stats().

◆ overload_happened_recently()

static bool overload_happened_recently	(	time_t	overload_time,
		int	n_hours
	)

static

Return true if this overload happened within the last n_hours.

Definition at line 391 of file rephist.c.

◆ parse_possibly_bad_iso_time()

static int parse_possibly_bad_iso_time	(	const char *	s,
		time_t *	time_out
	)

static

Parse the ISO-formatted time in s into *time_out, but round any pre-1970 date to Jan 1, 1970.

Definition at line 1084 of file rephist.c.

◆ rep_hist_add_buffer_stats()

void rep_hist_add_buffer_stats	(	double	mean_num_cells_in_queue,
		double	mean_time_cells_in_queue,
		uint32_t	processed_cells
	)

Remember cell statistics mean_num_cells_in_queue, mean_time_cells_in_queue, and processed_cells of a circuit.

Definition at line 1620 of file rephist.c.

Referenced by rep_hist_buffer_stats_add_circ().

◆ rep_hist_buffer_stats_add_circ()

void rep_hist_buffer_stats_add_circ	(	circuit_t *	circ,
		time_t	end_of_interval
	)

Remember cell statistics for circuit circ at time end_of_interval and reset cell counters in case the circuit remains open in the next measurement interval.

Definition at line 1639 of file rephist.c.

◆ rep_hist_buffer_stats_init()

void rep_hist_buffer_stats_init ( time_t now )

Initialize buffer stats.

Definition at line 1597 of file rephist.c.

◆ rep_hist_buffer_stats_term()

void rep_hist_buffer_stats_term ( void )

Stop collecting cell stats in a way that we can re-start doing so in rep_hist_buffer_stats_init().

Definition at line 1689 of file rephist.c.

◆ rep_hist_buffer_stats_write()

time_t rep_hist_buffer_stats_write ( time_t now )

If 24 hours have passed since the beginning of the current buffer stats period, write buffer stats to $DATADIR/stats/buffer-stats (possibly overwriting an existing file) and reset counters. Return when we would next want to write buffer stats or 0 if we never want to write.

Definition at line 1814 of file rephist.c.

◆ rep_hist_desc_stats_init()

void rep_hist_desc_stats_init ( time_t now )

Initialize descriptor stats.

Definition at line 1861 of file rephist.c.

Referenced by rep_hist_reset_desc_stats().

◆ rep_hist_desc_stats_term()

void rep_hist_desc_stats_term ( void )

Stop collecting served descs stats, so that rep_hist_desc_stats_init() is safe to be called again.

Definition at line 1884 of file rephist.c.

Referenced by rep_hist_reset_desc_stats().

◆ rep_hist_desc_stats_write()

time_t rep_hist_desc_stats_write ( time_t now )

If WRITE_STATS_INTERVAL seconds have passed since the beginning of the current served desc stats interval, write the stats to $DATADIR/stats/served-desc-stats (possibly appending to an existing file) and reset the state for the next interval. Return when we would next want to write served desc stats or 0 if we won't want to write.

Definition at line 1951 of file rephist.c.

◆ rep_hist_downrate_old_runs()

time_t rep_hist_downrate_old_runs ( time_t now )

Helper: Discount all old MTBF data, if it is time to do so. Return the time at which we should next discount MTBF data.

Definition at line 718 of file rephist.c.

Referenced by downrate_stability_callback().

◆ rep_hist_dump_stats()

void rep_hist_dump_stats	(	time_t	now,
		int	severity
	)

Log all the reliability data we have remembered, with the chosen severity.

Definition at line 890 of file rephist.c.

◆ rep_hist_exit_stats_init()

void rep_hist_exit_stats_init ( time_t now )

Initialize exit port stats.

Definition at line 1340 of file rephist.c.

◆ rep_hist_exit_stats_term()

void rep_hist_exit_stats_term ( void )

Stop collecting exit port stats in a way that we can re-start doing so in rep_hist_exit_stats_init().

Definition at line 1361 of file rephist.c.

◆ rep_hist_exit_stats_write()

time_t rep_hist_exit_stats_write ( time_t now )

If 24 hours have passed since the beginning of the current exit port stats period, write exit stats to $DATADIR/stats/exit-stats (possibly overwriting an existing file) and reset counters. Return when we would next want to write exit stats or 0 if we never want to write.

Definition at line 1537 of file rephist.c.

◆ rep_hist_format_buffer_stats()

char* rep_hist_format_buffer_stats ( time_t now )

Return a newly allocated string containing the buffer statistics until now, or NULL if we're not collecting buffer stats. Caller must ensure start_of_buffer_stats_interval is in the past.

Definition at line 1711 of file rephist.c.

◆ rep_hist_format_desc_stats()

static char* rep_hist_format_desc_stats ( time_t now )

static

Helper for rep_hist_desc_stats_write(). Return a newly allocated string containing the served desc statistics until now, or NULL if we're not collecting served desc stats. Caller must ensure that now is not before start_of_served_descs_stats_interval.

Definition at line 1897 of file rephist.c.

◆ rep_hist_format_exit_stats()

char* rep_hist_format_exit_stats ( time_t now )

Return a newly allocated string containing the exit port statistics until now, or NULL if we're not collecting exit stats. Caller must ensure start_of_exit_stats_interval is in the past.

Definition at line 1382 of file rephist.c.

◆ rep_hist_format_hs_stats()

STATIC char* rep_hist_format_hs_stats	(	time_t	now,
		bool	is_v3
	)

Allocate and return a string containing hidden service stats that are meant to be placed in the extra-info descriptor.

Function works for both v2 and v3 stats depending on is_v3.

Definition at line 2318 of file rephist.c.

◆ rep_hist_free_all()

void rep_hist_free_all ( void )

Free all storage held by the OR/link history caches, by the bandwidth history arrays, by the port history, or by statistics .

Definition at line 2610 of file rephist.c.

◆ rep_hist_get_circuit_handshake_assigned()

int rep_hist_get_circuit_handshake_assigned ( uint16_t type )

Get the circuit handshake value that is assigned.

Definition at line 2045 of file rephist.c.

Referenced by getinfo_helper_rephist().

◆ rep_hist_get_circuit_handshake_dropped()

uint64_t rep_hist_get_circuit_handshake_dropped ( uint16_t type )

Get the circuit handshake value that is dropped.

Definition at line 2055 of file rephist.c.

◆ rep_hist_get_circuit_handshake_requested()

int rep_hist_get_circuit_handshake_requested ( uint16_t type )

Get the circuit handshake value that is requested.

Definition at line 2035 of file rephist.c.

◆ rep_hist_get_n_dns_error()

uint64_t rep_hist_get_n_dns_error	(	int	type,
		uint8_t	error
	)

Return the DNS error count for the given libevent DNS type and error code. The possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA.

Definition at line 272 of file rephist.c.

◆ rep_hist_get_n_dns_request()

uint64_t rep_hist_get_n_dns_request ( int type )

Return the total number of DNS request seen for the given libevent DNS record type. Possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA.

Definition at line 313 of file rephist.c.

◆ rep_hist_get_n_read_limit_reached()

uint64_t rep_hist_get_n_read_limit_reached ( void )

Return the stats_n_read_limit_reached counter.

Definition at line 405 of file rephist.c.

Referenced by fill_global_bw_limit_values().

◆ rep_hist_get_n_tcp_exhaustion()

uint64_t rep_hist_get_n_tcp_exhaustion ( void )

Return the total number of TCP exhaustion times we've reached.

Definition at line 530 of file rephist.c.

Referenced by fill_tcp_exhaustion_values().

◆ rep_hist_get_n_write_limit_reached()

uint64_t rep_hist_get_n_write_limit_reached ( void )

Return the stats_n_write_limit_reached counter.

Definition at line 412 of file rephist.c.

Referenced by fill_global_bw_limit_values().

◆ rep_hist_get_overload_general_line()

char* rep_hist_get_overload_general_line ( void )

Returns an allocated string for server descriptor for publising information on whether we are overloaded or not.

Definition at line 420 of file rephist.c.

◆ rep_hist_get_overload_stats_lines()

char* rep_hist_get_overload_stats_lines ( void )

Returns an allocated string for extra-info documents for publishing overload statistics.

Definition at line 438 of file rephist.c.

◆ rep_hist_get_padding_count_lines()

char* rep_hist_get_padding_count_lines ( void )

Returns an allocated string for extra-info documents for publishing padding statistics from the last 24 hour interval.

Definition at line 2540 of file rephist.c.

◆ rep_hist_get_stability()

double rep_hist_get_stability	(	const char *	id,
		time_t	when
	)

Return an estimated MTBF for the router whose identity digest is id. Return 0 if the router is unknown.

Definition at line 839 of file rephist.c.

Referenced by dirserv_thinks_router_is_unreliable().

◆ rep_hist_get_uptime()

long rep_hist_get_uptime	(	const char *	id,
		time_t	when
	)

Return how long the router whose identity digest is id has been reachable. Return 0 if the router is unknown or currently deemed unreachable.

Definition at line 826 of file rephist.c.

◆ rep_hist_get_weighted_fractional_uptime()

double rep_hist_get_weighted_fractional_uptime	(	const char *	id,
		time_t	when
	)

Return an estimated percent-of-time-online for the router whose identity digest is id. Return 0 if the router is unknown.

Definition at line 851 of file rephist.c.

◆ rep_hist_get_weighted_time_known()

long rep_hist_get_weighted_time_known	(	const char *	id,
		time_t	when
	)

Return a number representing how long we've known about the router whose digest is id. Return 0 if the router is unknown.

Be careful: this measure increases monotonically as we know the router for longer and longer, but it doesn't increase linearly.

Definition at line 867 of file rephist.c.

◆ rep_hist_have_measured_enough_stability()

int rep_hist_have_measured_enough_stability ( void )

Return true if we've been measuring MTBFs for long enough to pronounce on Stability.

Definition at line 879 of file rephist.c.

◆ rep_hist_hs_stats_init()

void rep_hist_hs_stats_init ( time_t now )

Generic HS stats code Initialize v2 and v3 hidden service statistics.

Definition at line 2262 of file rephist.c.

◆ rep_hist_hs_stats_term()

void rep_hist_hs_stats_term ( void )

Stop collecting hidden service stats in a way that we can re-start doing so in rep_hist_buffer_stats_init().

Definition at line 2282 of file rephist.c.

◆ rep_hist_hs_stats_write()

time_t rep_hist_hs_stats_write	(	time_t	now,
		bool	is_v3
	)

If 24 hours have passed since the beginning of the current HS stats period, write buffer stats to $DATADIR/stats/hidserv-v3-stats (possibly overwriting an existing file) and reset counters. Return when we would next want to write buffer stats or 0 if we never want to write. Function works for both v2 and v3 stats depending on is_v3.

Definition at line 2369 of file rephist.c.

◆ rep_hist_hsdir_stored_maybe_new_v3_onion()

void rep_hist_hsdir_stored_maybe_new_v3_onion ( const uint8_t * blinded_key )

We just received a new descriptor with blinded_key. See if we've seen this blinded key before, and if not add it to the stats.

Definition at line 2221 of file rephist.c.

◆ rep_hist_init()

void rep_hist_init ( void )

Initialize the static data structures for tracking history.

Definition at line 569 of file rephist.c.

Referenced by tor_init().

◆ rep_hist_load_mtbf_data()

int rep_hist_load_mtbf_data ( time_t now )

Load MTBF data from disk. Returns 0 on success or recoverable error, -1 on failure.

Definition at line 1125 of file rephist.c.

Referenced by options_act_dirauth_mtbf().

◆ rep_hist_log_circuit_handshake_stats()

void rep_hist_log_circuit_handshake_stats ( time_t now )

Log our onionskin statistics since the last time we were called.

Definition at line 2065 of file rephist.c.

◆ rep_hist_log_link_protocol_counts()

void rep_hist_log_link_protocol_counts ( void )

Log a heartbeat message explaining how many connections of each link protocol version we have used.

Definition at line 2586 of file rephist.c.

◆ rep_hist_make_router_pessimal()

void rep_hist_make_router_pessimal	(	const char *	id,
		time_t	when
	)

Mark a router with ID id as non-Running, and retroactively declare that it has never been running: give it no stability and no WFU.

Definition at line 704 of file rephist.c.

◆ rep_hist_note_circuit_handshake_assigned()

void rep_hist_note_circuit_handshake_assigned ( uint16_t type )

We've sent an onionskin (using the type handshake) to a cpuworker.

Definition at line 2018 of file rephist.c.

◆ rep_hist_note_circuit_handshake_dropped()

void rep_hist_note_circuit_handshake_dropped ( uint16_t type )

We've just drop an onionskin (using the type handshake) due to being overloaded.

Definition at line 2027 of file rephist.c.

◆ rep_hist_note_circuit_handshake_requested()

void rep_hist_note_circuit_handshake_requested ( uint16_t type )

A new onionskin (using the type handshake) has arrived.

Definition at line 2009 of file rephist.c.

◆ rep_hist_note_desc_served()

void rep_hist_note_desc_served ( const char * desc )

Called to note that we've served a given descriptor (by digest). Increments the count of descriptors served, and the number of times we've served this descriptor.

Definition at line 1982 of file rephist.c.

◆ rep_hist_note_dns_error()

void rep_hist_note_dns_error	(	int	type,
		uint8_t	error
	)

Note a DNS error for the given given libevent DNS record type and error code. Possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA.

Definition at line 325 of file rephist.c.

◆ rep_hist_note_dns_request()

void rep_hist_note_dns_request ( int type )

Note a DNS request for the given given libevent DNS record type.

Definition at line 378 of file rephist.c.

◆ rep_hist_note_exit_bytes()

void rep_hist_note_exit_bytes	(	uint16_t	port,
		size_t	num_written,
		size_t	num_read
	)

Note that we wrote num_written bytes and read num_read bytes to/from an exit connection to port.

Definition at line 1567 of file rephist.c.

Referenced by record_num_bytes_transferred_impl().

◆ rep_hist_note_exit_stream_opened()

void rep_hist_note_exit_stream_opened ( uint16_t port )

Note that we opened an exit stream to port.

Definition at line 1581 of file rephist.c.

◆ rep_hist_note_negotiated_link_proto()

void rep_hist_note_negotiated_link_proto	(	unsigned	link_proto,
		int	started_here
	)

Note that we negotiated link protocol version link_proto, on a connection that started here iff started_here is true.

Definition at line 2412 of file rephist.c.

◆ rep_hist_note_overload()

void rep_hist_note_overload ( overload_type_t overload )

Note down an overload event of type overload.

Definition at line 485 of file rephist.c.

Referenced by rep_hist_note_tcp_exhaustion(), and socket_failed_from_resource_exhaustion().

◆ rep_hist_note_router_reachable()

void rep_hist_note_router_reachable	(	const char *	id,
		const tor_addr_t *	at_addr,
		const uint16_t	at_port,
		time_t	when
	)

We have just decided that this router with identity digest id is reachable, meaning we will give it a "Running" flag for the next while.

Definition at line 577 of file rephist.c.

◆ rep_hist_note_router_unreachable()

void rep_hist_note_router_unreachable	(	const char *	id,
		time_t	when
	)

We have just decided that this router is unreachable, meaning we are taking away its "Running" flag.

Definition at line 650 of file rephist.c.

Referenced by rep_hist_make_router_pessimal().

◆ rep_hist_note_tcp_exhaustion()

void rep_hist_note_tcp_exhaustion ( void )

Note down that we've reached a TCP port exhaustion. This triggers an overload general event.

Definition at line 522 of file rephist.c.

Referenced by socket_failed_from_resource_exhaustion().

◆ rep_hist_padding_count_read()

void rep_hist_padding_count_read ( padding_type_t type )

Count a cell that we've received for padding overhead statistics.

RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be counted for PADDING_TYPE_TOTAL.

Definition at line 2470 of file rephist.c.

◆ rep_hist_padding_count_timers()

void rep_hist_padding_count_timers ( uint64_t num_timers )

Update the maximum count of total pending channel padding timers in this period.

Definition at line 2428 of file rephist.c.

◆ rep_hist_padding_count_write()

void rep_hist_padding_count_write ( padding_type_t type )

Count a cell that we sent for padding overhead statistics.

RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be counted for PADDING_TYPE_TOTAL.

Definition at line 2442 of file rephist.c.

◆ rep_hist_record_mtbf_data()

int rep_hist_record_mtbf_data	(	time_t	now,
		int	missing_means_down
	)

Write MTBF data to disk. Return 0 on success, negative on failure.

If missing_means_down, then if we're about to write an entry that is still considered up but isn't in our routerlist, consider it to be down.

Definition at line 962 of file rephist.c.

Referenced by save_stability_callback(), and tor_cleanup().

◆ rep_hist_reset_buffer_stats()

void rep_hist_reset_buffer_stats ( time_t now )

Clear history of circuit statistics and set the measurement interval start to now.

Definition at line 1697 of file rephist.c.

Referenced by rep_hist_buffer_stats_term().

◆ rep_hist_reset_desc_stats()

static void rep_hist_reset_desc_stats ( time_t now )

static

Reset served descs stats to empty, starting a new interval now.

Definition at line 1875 of file rephist.c.

◆ rep_hist_reset_exit_stats()

void rep_hist_reset_exit_stats ( time_t now )

Reset counters for exit port statistics.

Definition at line 1350 of file rephist.c.

◆ rep_hist_reset_hs_v2_stats()

static void rep_hist_reset_hs_v2_stats ( time_t now )

static

Clear history of hidden service statistics and set the measurement interval start to now.

Definition at line 2114 of file rephist.c.

Referenced by rep_hist_hs_stats_term().

◆ rep_hist_reset_hs_v3_stats()

static void rep_hist_reset_hs_v3_stats ( time_t now )

static

Clear history of hidden service statistics and set the measurement interval start to now.

Definition at line 2189 of file rephist.c.

Referenced by rep_hist_hs_stats_term().

◆ rep_hist_reset_padding_counts()

void rep_hist_reset_padding_counts ( void )

Reset our current padding statistics. Called once every 24 hours.

Definition at line 2495 of file rephist.c.

◆ rep_hist_seen_new_rp_cell()

void rep_hist_seen_new_rp_cell ( bool is_v2 )

We saw a new HS relay cell: count it! If is_v2 is set then it's a v2 RP cell, otherwise it's a v3.

Definition at line 2247 of file rephist.c.

◆ rep_history_clean()

void rep_history_clean ( time_t before )

Remove history info for routers/links that haven't changed since before.

Definition at line 929 of file rephist.c.

Referenced by clean_caches_callback(), and rep_hist_dump_stats().

◆ should_collect_v3_stats()

STATIC bool should_collect_v3_stats ( void )

Return true if it's a good time to collect v3 stats.

v3 stats have a strict stats collection period (from 12:00UTC to 12:00UTC on the real network). We don't want to collect statistics if (for example) we just booted and it's 03:00UTC; we will wait until 12:00UTC before we start collecting statistics to make sure that the final result represents the whole collection period. This behavior is controlled by rep_hist_hs_stats_init().

Definition at line 2213 of file rephist.c.

Referenced by rep_hist_hsdir_stored_maybe_new_v3_onion().

Variable Documentation

◆ circuits_for_buffer_stats

smartlist_t* circuits_for_buffer_stats = NULL

static

List of circ_buffer_stats_t.

Definition at line 1614 of file rephist.c.

Referenced by rep_hist_reset_buffer_stats().

◆ dns_A_stats

dns_stats_t dns_A_stats

static

DNS statistics store for each DNS record type for which tor supports only three at the moment: A, PTR and AAAA.

Definition at line 248 of file rephist.c.

◆ exit_bytes_read

uint64_t* exit_bytes_read = NULL

static

Number of bytes read in current period by exit port

Definition at line 1329 of file rephist.c.

Referenced by rep_hist_exit_stats_init(), rep_hist_exit_stats_term(), rep_hist_note_exit_bytes(), and rep_hist_reset_exit_stats().

◆ exit_bytes_written

uint64_t* exit_bytes_written = NULL

static

Number of bytes written in current period by exit port

Definition at line 1331 of file rephist.c.

Referenced by rep_hist_exit_stats_term(), rep_hist_note_exit_bytes(), and rep_hist_reset_exit_stats().

◆ exit_streams

uint32_t* exit_streams = NULL

static

Number of streams opened in current period by exit port

Definition at line 1333 of file rephist.c.

Referenced by rep_hist_exit_stats_term(), rep_hist_note_exit_stream_opened(), and rep_hist_reset_exit_stats().

◆ history_map

digestmap_t* history_map = NULL

static

Map from hex OR identity digest to or_history_t.

Definition at line 186 of file rephist.c.

Referenced by get_or_history(), rep_hist_downrate_old_runs(), rep_hist_dump_stats(), rep_hist_init(), and rep_history_clean().

◆ hs_v2_stats

hs_v2_stats_t* hs_v2_stats = NULL

static

Our v2 statistics structure singleton.

Definition at line 2085 of file rephist.c.

Referenced by rep_hist_hs_stats_init(), and rep_hist_reset_hs_v2_stats().

◆ hs_v3_stats

hs_v3_stats_t* hs_v3_stats = NULL

static

Our v3 statistics structure singleton.

Definition at line 2159 of file rephist.c.

Referenced by rep_hist_hs_stats_init(), rep_hist_hsdir_stored_maybe_new_v3_onion(), and rep_hist_reset_hs_v3_stats().

◆ n_bogus_times

int n_bogus_times = 0

static

How many bad times has parse_possibly_bad_iso_time() parsed?

Definition at line 1080 of file rephist.c.

◆ onion_handshakes_requested

STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0}

Internal statistics to track how many requests of each type of handshake we've received, and how many we've assigned to cpuworkers. Useful for seeing trends in cpu load.

Definition at line 2002 of file rephist.c.

◆ overload_stats

overload_stats_t overload_stats

static

Current state of overload stats

Definition at line 210 of file rephist.c.

◆ padding_current

padding_counts_t padding_current

static

Holds the current values of our padding statistics. It is not published until it is transferred to padding_published.

Definition at line 172 of file rephist.c.

Referenced by rep_hist_padding_count_timers(), and rep_hist_reset_padding_counts().

◆ padding_published

padding_counts_t padding_published

static

Remains fixed for a 24 hour period, and then is replaced by a redacted copy of padding_current

Definition at line 176 of file rephist.c.

Referenced by rep_hist_get_padding_count_lines().

◆ rephist_total_alloc

uint64_t rephist_total_alloc =0

Total number of bytes currently allocated in fields used by rephist.c.

Definition at line 94 of file rephist.c.

Referenced by dumpmemusage(), and free_or_history().

◆ rephist_total_num

uint32_t rephist_total_num =0

Number of or_history_t objects currently allocated.

Definition at line 96 of file rephist.c.

Referenced by dumpmemusage(), and free_or_history().

◆ served_descs

digestmap_t* served_descs = NULL

static

Digestmap to track which descriptors were downloaded this stats collection interval. It maps descriptor digest to pointers to 1, effectively turning this into a list.

Definition at line 1850 of file rephist.c.

Referenced by rep_hist_desc_stats_init(), and rep_hist_note_desc_served().

◆ stability_last_downrated

time_t stability_last_downrated = 0

static

When did we last multiply all routers' weighted_run_length and total_run_weights by STABILITY_ALPHA?

Definition at line 180 of file rephist.c.

Referenced by rep_hist_downrate_old_runs().

◆ start_of_buffer_stats_interval

time_t start_of_buffer_stats_interval

static

Start of the current buffer stats interval or 0 if we're not collecting buffer statistics.

Definition at line 1593 of file rephist.c.

Referenced by rep_hist_add_buffer_stats(), rep_hist_buffer_stats_add_circ(), rep_hist_buffer_stats_init(), rep_hist_buffer_stats_write(), and rep_hist_reset_buffer_stats().

◆ start_of_exit_stats_interval

time_t start_of_exit_stats_interval

static

Start time of exit stats or 0 if we're not collecting exit stats.

Definition at line 1336 of file rephist.c.

Referenced by rep_hist_exit_stats_init(), rep_hist_exit_stats_term(), rep_hist_exit_stats_write(), rep_hist_note_exit_bytes(), rep_hist_note_exit_stream_opened(), and rep_hist_reset_exit_stats().

◆ start_of_hs_v2_stats_interval

time_t start_of_hs_v2_stats_interval

static

Start of the current hidden service stats interval or 0 if we're not collecting hidden service statistics.

Definition at line 2082 of file rephist.c.

Referenced by rep_hist_hs_stats_init(), rep_hist_hs_stats_write(), and rep_hist_reset_hs_v2_stats().

◆ start_of_hs_v3_stats_interval

time_t start_of_hs_v3_stats_interval

static

Start of the current hidden service stats interval or 0 if we're not collecting hidden service statistics.

This is particularly important for v3 statistics since this variable controls the start time of initial v3 stats collection. It's initialized by rep_hist_hs_stats_init() to the next time period start (i.e. 12:00UTC), and should_collect_v3_stats() ensures that functions that collect v3 stats do not do so sooner than that.

Collecting stats from 12:00UTC to 12:00UTC is extremely important for v3 stats because rep_hist_hsdir_stored_maybe_new_v3_onion() uses the blinded key of each onion service as its double-counting index. Onion services rotate their descriptor at around 00:00UTC which means that their blinded key also changes around that time. However the precise time that onion services rotate their descriptors is actually when they fetch a new 00:00UTC consensus and that happens at a random time (e.g. it can even happen at 02:00UTC). This means that if we started keeping v3 stats at around 00:00UTC we wouldn't be able to tell when onion services change their blinded key and hence we would double count an unpredictable amount of them (for example, if an onion service fetches the 00:00UTC consensus at 01:00UTC it would upload to its old HSDir at 00:45UTC, and then to a different HSDir at 01:50UTC).

For this reason, we start collecting statistics at 12:00UTC. This way we know that by the time we stop collecting statistics for that time period 24 hours later, all the onion services have switched to their new blinded key. This way we can predict much better how much double counting has been performed.

Definition at line 2156 of file rephist.c.

Referenced by rep_hist_hs_stats_init(), rep_hist_hs_stats_write(), and should_collect_v3_stats().

◆ start_of_served_descs_stats_interval

time_t start_of_served_descs_stats_interval

static

Start time of served descs stats or 0 if we're not collecting those.

Definition at line 1857 of file rephist.c.

Referenced by rep_hist_desc_stats_write().

◆ stats_n_read_limit_reached

uint64_t stats_n_read_limit_reached = 0

static

Counters to count the number of times we've reached an overload for the global connection read/write limit. Reported on the MetricsPort.

Definition at line 214 of file rephist.c.

Referenced by rep_hist_get_n_read_limit_reached().

◆ stats_n_tcp_exhaustion

uint64_t stats_n_tcp_exhaustion = 0

static

Total number of times we've reached TCP port exhaustion.

Definition at line 218 of file rephist.c.

Referenced by rep_hist_get_n_tcp_exhaustion(), and rep_hist_note_tcp_exhaustion().

◆ total_descriptor_downloads

unsigned long total_descriptor_downloads

static

Number of how many descriptors were downloaded in total during this interval.

Definition at line 1854 of file rephist.c.

Referenced by rep_hist_note_desc_served().

Data Structures

Macros

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ EXIT_STATS_NUM_PORTS

◆ EXIT_STATS_ROUND_UP_BYTES

◆ EXIT_STATS_ROUND_UP_STREAMS

◆ EXIT_STATS_TOP_N_PORTS

◆ MIN_CELL_COUNTS_TO_PUBLISH

◆ REND_CELLS_DELTA_F

◆ ROUND_AND_SET_COUNT

◆ SET_TO_START_OF_HOUR

◆ STABILITY_ALPHA

◆ STABILITY_EPSILON

◆ STABILITY_INTERVAL

Function Documentation

◆ buffer_stats_compare_entries_()

◆ compare_int_()

◆ correct_time()

◆ find_next_with()

◆ free_or_history()

◆ get_dns_stats_by_type()

◆ get_or_history()

◆ get_stability()

◆ get_total_weighted_time()

◆ get_weighted_fractional_uptime()

◆ hs_v2_stats_free_()

◆ hs_v2_stats_new()

◆ hs_v3_stats_free_()

◆ hs_v3_stats_new()

◆ overload_happened_recently()

◆ parse_possibly_bad_iso_time()

◆ rep_hist_add_buffer_stats()

◆ rep_hist_buffer_stats_add_circ()

◆ rep_hist_buffer_stats_init()

◆ rep_hist_buffer_stats_term()

◆ rep_hist_buffer_stats_write()

◆ rep_hist_desc_stats_init()

◆ rep_hist_desc_stats_term()

◆ rep_hist_desc_stats_write()

◆ rep_hist_downrate_old_runs()

◆ rep_hist_dump_stats()

◆ rep_hist_exit_stats_init()

◆ rep_hist_exit_stats_term()

◆ rep_hist_exit_stats_write()

◆ rep_hist_format_buffer_stats()

◆ rep_hist_format_desc_stats()

◆ rep_hist_format_exit_stats()

◆ rep_hist_format_hs_stats()

◆ rep_hist_free_all()

◆ rep_hist_get_circuit_handshake_assigned()

◆ rep_hist_get_circuit_handshake_dropped()

◆ rep_hist_get_circuit_handshake_requested()

◆ rep_hist_get_n_dns_error()

◆ rep_hist_get_n_dns_request()

◆ rep_hist_get_n_read_limit_reached()

◆ rep_hist_get_n_tcp_exhaustion()

◆ rep_hist_get_n_write_limit_reached()

◆ rep_hist_get_overload_general_line()

◆ rep_hist_get_overload_stats_lines()

◆ rep_hist_get_padding_count_lines()

◆ rep_hist_get_stability()

◆ rep_hist_get_uptime()

◆ rep_hist_get_weighted_fractional_uptime()

◆ rep_hist_get_weighted_time_known()

◆ rep_hist_have_measured_enough_stability()

◆ rep_hist_hs_stats_init()

◆ rep_hist_hs_stats_term()

◆ rep_hist_hs_stats_write()

◆ rep_hist_hsdir_stored_maybe_new_v3_onion()

◆ rep_hist_init()

◆ rep_hist_load_mtbf_data()

◆ rep_hist_log_circuit_handshake_stats()

◆ rep_hist_log_link_protocol_counts()

◆ rep_hist_make_router_pessimal()

◆ rep_hist_note_circuit_handshake_assigned()

◆ rep_hist_note_circuit_handshake_dropped()

◆ rep_hist_note_circuit_handshake_requested()