tor  0.4.2.0-alpha-dev
Data Structures | Macros | Typedefs | Functions
storagedir.c File Reference
#include "lib/fs/storagedir.h"
#include "lib/container/smartlist.h"
#include "lib/encoding/confline.h"
#include "lib/fs/dir.h"
#include "lib/fs/files.h"
#include "lib/fs/mmap.h"
#include "lib/log/escape.h"
#include "lib/log/log.h"
#include "lib/log/util_bug.h"
#include "lib/malloc/malloc.h"
#include "lib/memarea/memarea.h"
#include "lib/sandbox/sandbox.h"
#include "lib/string/printf.h"
#include "lib/string/util_string.h"
#include <stdlib.h>
#include <errno.h>
#include <string.h>

Go to the source code of this file.

Data Structures

struct  storage_dir_t
 
struct  shrinking_dir_entry_t
 

Macros

#define FNAME_MIN_NUM   1000
 

Typedefs

typedef struct shrinking_dir_entry_t shrinking_dir_entry_t
 

Functions

storage_dir_tstorage_dir_new (const char *dirname, int max_files)
 
void storage_dir_free_ (storage_dir_t *d)
 
int storage_dir_register_with_sandbox (storage_dir_t *d, sandbox_cfg_t **cfg)
 
static void storage_dir_clean_tmpfiles (storage_dir_t *d)
 
static int storage_dir_rescan (storage_dir_t *d)
 
const smartlist_tstorage_dir_list (storage_dir_t *d)
 
uint64_t storage_dir_get_usage (storage_dir_t *d)
 
tor_mmap_tstorage_dir_map (storage_dir_t *d, const char *fname)
 
uint8_t * storage_dir_read (storage_dir_t *d, const char *fname, int bin, size_t *sz_out)
 
static char * find_unused_fname (storage_dir_t *d)
 
static int storage_dir_save_chunks_to_file (storage_dir_t *d, const smartlist_t *chunks, int binary, char **fname_out)
 
int storage_dir_save_bytes_to_file (storage_dir_t *d, const uint8_t *data, size_t length, int binary, char **fname_out)
 
int storage_dir_save_string_to_file (storage_dir_t *d, const char *str, int binary, char **fname_out)
 
int storage_dir_save_labeled_to_file (storage_dir_t *d, const config_line_t *labels, const uint8_t *data, size_t length, char **fname_out)
 
tor_mmap_tstorage_dir_map_labeled (storage_dir_t *dir, const char *fname, config_line_t **labels_out, const uint8_t **data_out, size_t *sz_out)
 
uint8_t * storage_dir_read_labeled (storage_dir_t *dir, const char *fname, config_line_t **labels_out, size_t *sz_out)
 
static void storage_dir_reduce_usage (storage_dir_t *d, uint64_t removed_file_size)
 
void storage_dir_remove_file (storage_dir_t *d, const char *fname)
 
static int shrinking_dir_entry_compare (const void *a_, const void *b_)
 
int storage_dir_shrink (storage_dir_t *d, uint64_t target_size, int min_to_remove)
 
int storage_dir_remove_all (storage_dir_t *d)
 
int storage_dir_get_max_files (storage_dir_t *d)
 

Detailed Description

An abstraction for a directory full of similar files.

Storagedirs are used by our consensus cache code, and may someday also get used for unparseable objects. A large part of the need for this type is to work around the limitations in our sandbox code, where all filenames need to be registered in advance.

Definition in file storagedir.c.

Typedef Documentation

◆ shrinking_dir_entry_t

Helper type: used to sort the members of storage directory by mtime.

Function Documentation

◆ find_unused_fname()

static char* find_unused_fname ( storage_dir_t d)
static

Helper: Find an unused filename within the directory

Definition at line 252 of file storagedir.c.

References storage_dir_t::contents, and storage_dir_rescan().

Referenced by storage_dir_save_chunks_to_file().

◆ shrinking_dir_entry_compare()

static int shrinking_dir_entry_compare ( const void *  a_,
const void *  b_ 
)
static

Helper: use with qsort to sort shrinking_dir_entry_t structs.

Definition at line 518 of file storagedir.c.

◆ storage_dir_clean_tmpfiles()

static void storage_dir_clean_tmpfiles ( storage_dir_t d)
static

Remove all files in d whose names end with ".tmp".

Requires that the contents field of d is set.

Definition at line 133 of file storagedir.c.

References storage_dir_t::contents, storage_dir_t::directory, sandbox_intern_string, SMARTLIST_FOREACH_BEGIN, strcmpend(), and tor_asprintf().

◆ storage_dir_free_()

void storage_dir_free_ ( storage_dir_t d)

Drop all in-RAM storage for d. Does not delete any files.

Definition at line 83 of file storagedir.c.

References storage_dir_t::contents, storage_dir_t::directory, SMARTLIST_FOREACH, and tor_free.

◆ storage_dir_get_max_files()

int storage_dir_get_max_files ( storage_dir_t d)

Return the largest number of non-temporary files we're willing to store in d.

Definition at line 603 of file storagedir.c.

References storage_dir_t::max_files.

◆ storage_dir_get_usage()

uint64_t storage_dir_get_usage ( storage_dir_t d)

Return the total number of bytes used for storage in d.

Definition at line 190 of file storagedir.c.

References storage_dir_t::directory, sandbox_intern_string, SMARTLIST_FOREACH_BEGIN, storage_dir_list(), tor_asprintf(), tor_free, storage_dir_t::usage, and storage_dir_t::usage_known.

Referenced by storage_dir_shrink().

◆ storage_dir_list()

const smartlist_t* storage_dir_list ( storage_dir_t d)

Return a smartlist containing the filenames within d.

Definition at line 179 of file storagedir.c.

References storage_dir_t::contents, and storage_dir_rescan().

Referenced by storage_dir_get_usage().

◆ storage_dir_map()

tor_mmap_t* storage_dir_map ( storage_dir_t d,
const char *  fname 
)

Mmap a specified file within d.

On failure, return NULL and set errno as for tor_mmap_file().

Definition at line 215 of file storagedir.c.

References storage_dir_t::directory, and tor_asprintf().

Referenced by storage_dir_map_labeled().

◆ storage_dir_map_labeled()

tor_mmap_t* storage_dir_map_labeled ( storage_dir_t dir,
const char *  fname,
config_line_t **  labels_out,
const uint8_t **  data_out,
size_t *  sz_out 
)

Map a file that was created with storage_dir_save_labeled_to_file(). On failure, return NULL. On success, write a set of newly allocated labels into *labels_out, a pointer to the data into *data_out, and the data's size into *sz_out. On success, also return a tor_mmap_t object whose contents should not be used – it needs to be kept around, though, for as long as data_out is going to be valid.

On failure, set errno as for tor_mmap_file() if the file was missing or empty, and set errno to EINVAL if the file was not in the labeled format expected.

Definition at line 399 of file storagedir.c.

References config_get_lines(), tor_mmap_t::data, tor_mmap_t::size, storage_dir_map(), and tor_assert().

Referenced by storage_dir_read_labeled().

◆ storage_dir_new()

storage_dir_t* storage_dir_new ( const char *  dirname,
int  max_files 
)

Create or open a new storage directory at dirname, with capacity for up to max_files files.

Definition at line 68 of file storagedir.c.

◆ storage_dir_read()

uint8_t* storage_dir_read ( storage_dir_t d,
const char *  fname,
int  bin,
size_t *  sz_out 
)

Read a file within d into a newly allocated buffer. Set *sz_out to its size.

Definition at line 230 of file storagedir.c.

References storage_dir_t::directory, RFTS_BIN, tor_asprintf(), tor_assert(), and tor_free.

◆ storage_dir_read_labeled()

uint8_t* storage_dir_read_labeled ( storage_dir_t dir,
const char *  fname,
config_line_t **  labels_out,
size_t *  sz_out 
)

As storage_dir_map_labeled, but return a new byte array containing the data.

Definition at line 435 of file storagedir.c.

References storage_dir_map_labeled().

◆ storage_dir_register_with_sandbox()

int storage_dir_register_with_sandbox ( storage_dir_t d,
sandbox_cfg_t **  cfg 
)

Tell the sandbox (if any) configured by cfg to allow the operations that d will need.

The presence of this function is why we need an upper limit on the number of files in a storage_dir_t: we need to approve file operations one by one.

Definition at line 104 of file storagedir.c.

◆ storage_dir_remove_all()

int storage_dir_remove_all ( storage_dir_t d)

Remove all files in d.

Definition at line 593 of file storagedir.c.

References storage_dir_t::max_files, and storage_dir_shrink().

◆ storage_dir_remove_file()

void storage_dir_remove_file ( storage_dir_t d,
const char *  fname 
)

Remove the file called fname from d.

Definition at line 480 of file storagedir.c.

References storage_dir_t::directory, sandbox_intern_string, tor_asprintf(), and storage_dir_t::usage_known.

◆ storage_dir_rescan()

static int storage_dir_rescan ( storage_dir_t d)
static

Re-scan the directory d to learn its contents.

Definition at line 160 of file storagedir.c.

References storage_dir_t::contents, SMARTLIST_FOREACH, and tor_free.

Referenced by find_unused_fname(), storage_dir_list(), and storage_dir_shrink().

◆ storage_dir_save_bytes_to_file()

int storage_dir_save_bytes_to_file ( storage_dir_t d,
const uint8_t *  data,
size_t  length,
int  binary,
char **  fname_out 
)

Try to write the length bytes at data into a new file in d. On success, return 0 and set *fname_out to a newly allocated string containing the filename. On failure, return -1.

Definition at line 311 of file storagedir.c.

References smartlist_add(), and storage_dir_save_chunks_to_file().

Referenced by storage_dir_save_string_to_file().

◆ storage_dir_save_chunks_to_file()

static int storage_dir_save_chunks_to_file ( storage_dir_t d,
const smartlist_t chunks,
int  binary,
char **  fname_out 
)
static

Helper: As storage_dir_save_bytes_to_file, but store a smartlist of sized_chunk_t rather than a single byte array.

Definition at line 275 of file storagedir.c.

References storage_dir_t::directory, find_unused_fname(), SMARTLIST_FOREACH, and tor_asprintf().

Referenced by storage_dir_save_bytes_to_file(), and storage_dir_save_labeled_to_file().

◆ storage_dir_save_labeled_to_file()

int storage_dir_save_labeled_to_file ( storage_dir_t d,
const config_line_t labels,
const uint8_t *  data,
size_t  length,
char **  fname_out 
)

As storage_dir_save_bytes_to_file, but associates the data with the key-value pairs in labels. Files stored in this format can be recovered with storage_dir_map_labeled() or storage_dir_read_labeled().

Definition at line 345 of file storagedir.c.

References memarea_alloc(), memarea_new(), smartlist_add(), storage_dir_save_chunks_to_file(), and tor_snprintf().

◆ storage_dir_save_string_to_file()

int storage_dir_save_string_to_file ( storage_dir_t d,
const char *  str,
int  binary,
char **  fname_out 
)

As storage_dir_save_bytes_to_file, but saves a NUL-terminated string str.

Definition at line 330 of file storagedir.c.

References storage_dir_save_bytes_to_file().

◆ storage_dir_shrink()

int storage_dir_shrink ( storage_dir_t d,
uint64_t  target_size,
int  min_to_remove 
)

Try to free space by removing the oldest files in d. Delete until no more than target_size bytes are left, and at least min_to_remove files have been removed... or until there is nothing left to remove.

Return 0 on success; -1 on failure.

Definition at line 540 of file storagedir.c.

References storage_dir_get_usage(), storage_dir_rescan(), storage_dir_t::usage, and storage_dir_t::usage_known.

Referenced by storage_dir_remove_all().