storagedir.c File Reference

An abstraction for a directory full of similar files. More...

#include "lib/fs/storagedir.h"
#include "lib/container/smartlist.h"
#include "lib/encoding/confline.h"
#include "lib/fs/dir.h"
#include "lib/fs/files.h"
#include "lib/fs/mmap.h"
#include "lib/log/escape.h"
#include "lib/log/log.h"
#include "lib/log/util_bug.h"
#include "lib/malloc/malloc.h"
#include "lib/memarea/memarea.h"
#include "lib/sandbox/sandbox.h"
#include "lib/string/printf.h"
#include "lib/string/util_string.h"
#include <stdlib.h>
#include <errno.h>
#include <string.h>

Go to the source code of this file.

Data Structures
struct	storage_dir_t
struct	shrinking_dir_entry_t

Macros
#define	FNAME_MIN_NUM   1000

Functions
storage_dir_t *	storage_dir_new (const char *dirname, int max_files)
void	storage_dir_free_ (storage_dir_t *d)
int	storage_dir_register_with_sandbox (storage_dir_t *d, sandbox_cfg_t **cfg)
static void	storage_dir_clean_tmpfiles (storage_dir_t *d)
static int	storage_dir_rescan (storage_dir_t *d)
const smartlist_t *	storage_dir_list (storage_dir_t *d)
uint64_t	storage_dir_get_usage (storage_dir_t *d)
tor_mmap_t *	storage_dir_map (storage_dir_t *d, const char *fname)
uint8_t *	storage_dir_read (storage_dir_t *d, const char *fname, int bin, size_t *sz_out)
static char *	find_unused_fname (storage_dir_t *d)
static int	storage_dir_save_chunks_to_file (storage_dir_t *d, const smartlist_t *chunks, int binary, char **fname_out)
int	storage_dir_save_bytes_to_file (storage_dir_t *d, const uint8_t *data, size_t length, int binary, char **fname_out)
int	storage_dir_save_string_to_file (storage_dir_t *d, const char *str, int binary, char **fname_out)
int	storage_dir_save_labeled_to_file (storage_dir_t *d, const config_line_t *labels, const uint8_t *data, size_t length, char **fname_out)
tor_mmap_t *	storage_dir_map_labeled (storage_dir_t *dir, const char *fname, config_line_t **labels_out, const uint8_t **data_out, size_t *sz_out)
uint8_t *	storage_dir_read_labeled (storage_dir_t *dir, const char *fname, config_line_t **labels_out, size_t *sz_out)
static void	storage_dir_reduce_usage (storage_dir_t *d, uint64_t removed_file_size)
void	storage_dir_remove_file (storage_dir_t *d, const char *fname)
static int	shrinking_dir_entry_compare (const void *a_, const void *b_)
int	storage_dir_shrink (storage_dir_t *d, uint64_t target_size, int min_to_remove)
int	storage_dir_remove_all (storage_dir_t *d)
int	storage_dir_get_max_files (storage_dir_t *d)

Detailed Description

An abstraction for a directory full of similar files.

Storagedirs are used by our consensus cache code, and may someday also get used for unparseable objects. A large part of the need for this type is to work around the limitations in our sandbox code, where all filenames need to be registered in advance.

Definition in file storagedir.c.

Function Documentation

find_unused_fname()

static char* find_unused_fname ( storage_dir_t *  d )

static

Helper: Find an unused filename within the directory

Definition at line 252 of file storagedir.c.

Referenced by storage_dir_save_chunks_to_file().

shrinking_dir_entry_compare()

static int shrinking_dir_entry_compare ( const void *  a_, const void *  b_  )

static

Helper: use with qsort to sort shrinking_dir_entry_t structs.

Definition at line 518 of file storagedir.c.

storage_dir_clean_tmpfiles()

static void storage_dir_clean_tmpfiles ( storage_dir_t *  d )

static

Remove all files in d whose names end with ".tmp".

Requires that the contents field of d is set.

Definition at line 133 of file storagedir.c.

storage_dir_free_()

void storage_dir_free_

(

storage_dir_t *

d

)

Drop all in-RAM storage for d. Does not delete any files.

Definition at line 83 of file storagedir.c.

storage_dir_get_max_files()

int storage_dir_get_max_files

(

storage_dir_t *

d

)

Return the largest number of non-temporary files we're willing to store in d.

Definition at line 603 of file storagedir.c.

storage_dir_get_usage()

uint64_t storage_dir_get_usage

(

storage_dir_t *

d

)

Return the total number of bytes used for storage in d.

Definition at line 190 of file storagedir.c.

Referenced by storage_dir_shrink().

storage_dir_list()

const smartlist_t* storage_dir_list

(

storage_dir_t *

d

)

Return a smartlist containing the filenames within d.

Definition at line 179 of file storagedir.c.

Referenced by consensus_cache_rescan(), and storage_dir_get_usage().

storage_dir_map()

tor_mmap_t* storage_dir_map	(	storage_dir_t *	d,
		const char *	fname
	)

Mmap a specified file within d.

On failure, return NULL and set errno as for tor_mmap_file().

Definition at line 215 of file storagedir.c.

Referenced by storage_dir_map_labeled().

storage_dir_map_labeled()

tor_mmap_t* storage_dir_map_labeled	(	storage_dir_t *	dir,
		const char *	fname,
		config_line_t **	labels_out,
		const uint8_t **	data_out,
		size_t *	sz_out
	)

Map a file that was created with storage_dir_save_labeled_to_file(). On failure, return NULL. On success, write a set of newly allocated labels into *labels_out, a pointer to the data into *data_out, and the data's size into *sz_out. On success, also return a tor_mmap_t object whose contents should not be used – it needs to be kept around, though, for as long as data_out is going to be valid.

On failure, set errno as for tor_mmap_file() if the file was missing or empty, and set errno to EINVAL if the file was not in the labeled format expected.

Definition at line 399 of file storagedir.c.

Referenced by consensus_cache_entry_map(), consensus_cache_rescan(), and storage_dir_read_labeled().

storage_dir_new()

storage_dir_t* storage_dir_new	(	const char *	dirname,
		int	max_files
	)

Create or open a new storage directory at dirname, with capacity for up to max_files files.

Definition at line 68 of file storagedir.c.

storage_dir_read()

uint8_t* storage_dir_read	(	storage_dir_t *	d,
		const char *	fname,
		int	bin,
		size_t *	sz_out
	)

Read a file within d into a newly allocated buffer. Set *sz_out to its size.

Definition at line 230 of file storagedir.c.

storage_dir_read_labeled()

uint8_t* storage_dir_read_labeled	(	storage_dir_t *	dir,
		const char *	fname,
		config_line_t **	labels_out,
		size_t *	sz_out
	)

As storage_dir_map_labeled, but return a new byte array containing the data.

Definition at line 435 of file storagedir.c.

storage_dir_register_with_sandbox()

int storage_dir_register_with_sandbox	(	storage_dir_t *	d,
		sandbox_cfg_t **	cfg
	)

Tell the sandbox (if any) configured by cfg to allow the operations that d will need.

The presence of this function is why we need an upper limit on the number of files in a storage_dir_t: we need to approve file operations one by one.

Definition at line 104 of file storagedir.c.

Referenced by consensus_cache_register_with_sandbox().

storage_dir_remove_all()

int storage_dir_remove_all

(

storage_dir_t *

d

)

Remove all files in d.

Definition at line 593 of file storagedir.c.

storage_dir_remove_file()

void storage_dir_remove_file	(	storage_dir_t *	d,
		const char *	fname
	)

Remove the file called fname from d.

Definition at line 480 of file storagedir.c.

storage_dir_rescan()

static int storage_dir_rescan ( storage_dir_t *  d )

static

Re-scan the directory d to learn its contents.

Definition at line 160 of file storagedir.c.

Referenced by find_unused_fname(), storage_dir_list(), and storage_dir_shrink().

storage_dir_save_bytes_to_file()

int storage_dir_save_bytes_to_file	(	storage_dir_t *	d,
		const uint8_t *	data,
		size_t	length,
		int	binary,
		char **	fname_out
	)

Try to write the length bytes at data into a new file in d. On success, return 0 and set *fname_out to a newly allocated string containing the filename. On failure, return -1.

Definition at line 311 of file storagedir.c.

Referenced by storage_dir_save_string_to_file().

storage_dir_save_chunks_to_file()

static int storage_dir_save_chunks_to_file ( storage_dir_t *  d, const smartlist_t *  chunks, int  binary, char **  fname_out  )

static

Helper: As storage_dir_save_bytes_to_file, but store a smartlist of sized_chunk_t rather than a single byte array.

Definition at line 275 of file storagedir.c.

Referenced by storage_dir_save_bytes_to_file(), and storage_dir_save_labeled_to_file().

storage_dir_save_labeled_to_file()

int storage_dir_save_labeled_to_file	(	storage_dir_t *	d,
		const config_line_t *	labels,
		const uint8_t *	data,
		size_t	length,
		char **	fname_out
	)

As storage_dir_save_bytes_to_file, but associates the data with the key-value pairs in labels. Files stored in this format can be recovered with storage_dir_map_labeled() or storage_dir_read_labeled().

Definition at line 345 of file storagedir.c.

Referenced by consensus_cache_add().

storage_dir_save_string_to_file()

int storage_dir_save_string_to_file	(	storage_dir_t *	d,
		const char *	str,
		int	binary,
		char **	fname_out
	)

As storage_dir_save_bytes_to_file, but saves a NUL-terminated string str.

Definition at line 330 of file storagedir.c.

storage_dir_shrink()

int storage_dir_shrink	(	storage_dir_t *	d,
		uint64_t	target_size,
		int	min_to_remove
	)

Try to free space by removing the oldest files in d. Delete until no more than target_size bytes are left, and at least min_to_remove files have been removed... or until there is nothing left to remove.

Return 0 on success; -1 on failure.

Definition at line 540 of file storagedir.c.

Referenced by storage_dir_remove_all().