From 56e54c8c75d59a8e5072451941aa56c727d99d26 Mon Sep 17 00:00:00 2001 From: Andre Noll Date: Fri, 6 Mar 2020 14:26:39 +0100 Subject: [PATCH] Add para_upgrade_db. This new executable is required to convert existing paraslash databases to the new format. Only the layout of the audio file table changes due to the switch from sha1 to sha256. The command creates and opens a new audio file table and copies over each row, using an arbitrary (incorrect) hash value for the values in hash column of the destination table. After all rows have been copied in this way, the old table is deleted and the database directory is renamed from afs-database-0.4 to afs-database-0.7, which will be the default path in paraslash-0.7. Subsequent patches will modify para_server to load the database from the new path and use sha256 instead of sha1 for the hash that identifies the audio file. The user must then start the thusly patched para_server and force-add all audio files to correct the hashes. This approach keeps para_upgrade_db minimal and shortens its running time. --- Makefile.in | 1 + Makefile.real | 5 +- configure.ac | 14 +- m4/lls/upgrade_db.suite.m4 | 33 ++++ upgrade_db.c | 382 +++++++++++++++++++++++++++++++++++++ 5 files changed, 433 insertions(+), 2 deletions(-) create mode 100644 m4/lls/upgrade_db.suite.m4 create mode 100644 upgrade_db.c diff --git a/Makefile.in b/Makefile.in index d4a83a77..f9efa7de 100644 --- a/Makefile.in +++ b/Makefile.in @@ -23,6 +23,7 @@ audiod_objs := @audiod_objs@ audioc_objs := @audioc_objs@ mixer_objs := @mixer_objs@ server_objs := @server_objs@ +upgrade_db_objs := @upgrade_db_objs@ write_objs := @write_objs@ afh_objs := @afh_objs@ play_objs := @play_objs@ diff --git a/Makefile.real b/Makefile.real index b88de225..140d8f31 100644 --- a/Makefile.real +++ b/Makefile.real @@ -55,6 +55,7 @@ gui_objs += gui.lsg.o play_objs += $(addsuffix _cmd.lsg.o, recv filter play write) play.lsg.o recv_objs += recv_cmd.lsg.o recv.lsg.o server_objs += server_cmd.lsg.o server.lsg.o +upgrade_db_objs += upgrade_db.lsg.o write_objs += write_cmd.lsg.o write.lsg.o cmd_suites := $(addsuffix _cmd, audiod server play recv filter write) @@ -77,6 +78,7 @@ audiod_objs := $(addprefix $(object_dir)/, $(audiod_objs)) audioc_objs := $(addprefix $(object_dir)/, $(audioc_objs)) mixer_objs := $(addprefix $(object_dir)/, $(mixer_objs)) server_objs := $(addprefix $(object_dir)/, $(server_objs)) +upgrade_db_objs := $(addprefix $(object_dir)/, $(upgrade_db_objs)) write_objs := $(addprefix $(object_dir)/, $(write_objs)) afh_objs := $(addprefix $(object_dir)/, $(afh_objs)) play_objs := $(addprefix $(object_dir)/, $(play_objs)) @@ -257,7 +259,7 @@ para_recv para_afh para_play para_server: LDFLAGS += $(id3tag_ldflags) para_write para_play para_audiod \ : LDFLAGS += $(ao_ldflags) $(pthread_ldflags) para_client para_audioc para_play : LDFLAGS += $(readline_ldflags) -para_server: LDFLAGS += $(osl_ldflags) +para_server para_upgrade_db: LDFLAGS += $(osl_ldflags) para_gui: LDFLAGS += $(curses_ldflags) para_server \ para_client \ @@ -294,6 +296,7 @@ para_gui \ para_play \ para_recv \ para_server \ +para_upgrade_db \ para_write \ : LDFLAGS += $(lopsub_ldflags) diff --git a/configure.ac b/configure.ac index f518f89a..42860bbb 100644 --- a/configure.ac +++ b/configure.ac @@ -377,7 +377,7 @@ UNSTASH_FLAGS if test -n "$CRYPTOLIB" && test $HAVE_OSL = yes && test -n "$BISON" && \ test -n "$FLEX"; then build_server="yes" - executables="$executables server" + executables="$executables server upgrade_db" server_errlist_objs=" server afh_common @@ -437,6 +437,17 @@ if test -n "$CRYPTOLIB" && test $HAVE_OSL = yes && test -n "$BISON" && \ else build_server="no" fi +############################################################# upgrade_db +upgrade_db_objs=' + crypt_common + exec + fd + string + upgrade_db + version + base64 +' +AC_SUBST(upgrade_db_objs, add_dot_o($upgrade_db_objs)) ############################################################# client if test -n "$CRYPTOLIB"; then build_client="yes" @@ -855,6 +866,7 @@ id3 version 2 support: $HAVE_ID3TAG faad: $HAVE_FAAD audio format handlers: $audio_format_handlers +exe: $executables para_server: $build_server para_gui: $build_gui para_mixer: $build_mixer diff --git a/m4/lls/upgrade_db.suite.m4 b/m4/lls/upgrade_db.suite.m4 new file mode 100644 index 00000000..7f46b749 --- /dev/null +++ b/m4/lls/upgrade_db.suite.m4 @@ -0,0 +1,33 @@ +m4_define(PROGRAM, para_upgrade_db) +[suite upgrade_db] +version-string = GIT_VERSION() +[supercommand para_upgrade_db] + purpose = upgrade the paraslash database to version 0.7 + [description] + The database format changes with paraslash-0.7.0. This program converts + the database from the older 0.4 format that was used in paraslash 0.4.x + through 0.6.x. In has to be executed only once. + [/description] + m4_include(common-option-section.m4) + m4_include(help.m4) + m4_include(detailed-help.m4) + m4_include(version.m4) + m4_include(loglevel.m4) + [option src-database-dir] + summary = location of the old afs database + arg_info = required_arg + arg_type = string + typestr = directory + [help] + The directory which contains the database to be converted. The default + is ~/.paraslash/afs_database-0.4. + [/help] + [option dst-database-dir] + summary = location of the new afs database + arg_info = required_arg + arg_type = string + typestr = directory + [help] + The directory which contains the converted database after the program + has terminated. The default is ~/.paraslash/afs_database-0.7. + [/help] diff --git a/upgrade_db.c b/upgrade_db.c new file mode 100644 index 00000000..bf81a61a --- /dev/null +++ b/upgrade_db.c @@ -0,0 +1,382 @@ +#include +#include +#include + +#include "upgrade_db.lsg.h" +#include "para.h" +#include "error.h" +#include "string.h" +#include "fd.h" +#include "crypt.h" +#include "version.h" + +#define CMD_PTR (lls_cmd(0, upgrade_db_suite)) +#define OPT_RESULT(_name, _lpr) \ + (lls_opt_result(LSG_UPGRADE_DB_PARA_UPGRADE_DB_OPT_ ## _name, lpr)) +#define OPT_GIVEN(_name, _lpr) (lls_opt_given(OPT_RESULT(_name, _lpr))) +#define OPT_UINT32_VAL(_name, _lpr) (lls_uint32_val(0, OPT_RESULT(_name, _lpr))) +#define OPT_STRING_VAL(_name, _lpr) (lls_string_val(0, OPT_RESULT(_name, _lpr))) + +static int loglevel; +INIT_STDERR_LOGGING(loglevel); + +/** Array of error strings. */ +DEFINE_PARA_ERRLIST; + +static void handle_help_flag(struct lls_parse_result *lpr) +{ + char *help; + + if (OPT_GIVEN(DETAILED_HELP, lpr)) + help = lls_long_help(CMD_PTR); + else if (OPT_GIVEN(HELP, lpr)) + help = lls_short_help(CMD_PTR); + else + return; + printf("%s\n", help); + free(help); + exit(EXIT_SUCCESS); +} + +static struct stat *path_exists(const char *path) +{ + static struct stat sb; + + if (stat(path, &sb) < 0) + return NULL; + return &sb; +} + +static bool is_dir(const char *path) +{ + struct stat *sb = path_exists(path); + if (!sb) + return false; + return (sb->st_mode & S_IFMT) == S_IFDIR; +} + +__noreturn static void die(const char *msg) +{ + PARA_EMERG_LOG("%s\n", msg); + exit(EXIT_FAILURE); +} + +static int string_compare(const struct osl_object *obj1, const struct osl_object *obj2) +{ + const char *str1 = obj1->data; + const char *str2 = obj2->data; + return strncmp(str1, str2, PARA_MIN(obj1->size, obj2->size)); +} + +static char *src_db_dir, *dst_db_dir, *src_aft_dir, *dst_aft_dir; + +static void set_paths(const struct lls_parse_result *lpr) +{ + char *home = para_homedir(); + + if (OPT_GIVEN(SRC_DATABASE_DIR, lpr)) + src_db_dir = para_strdup(OPT_STRING_VAL(SRC_DATABASE_DIR, + lpr)); + else + src_db_dir = make_message( + "%s/.paraslash/afs_database-0.4", home); + if (OPT_GIVEN(DST_DATABASE_DIR, lpr)) + dst_db_dir = para_strdup(OPT_STRING_VAL(DST_DATABASE_DIR, + lpr)); + else + dst_db_dir = make_message( + "%s/.paraslash/afs_database-0.7", home); + free(home); + src_aft_dir = make_message("%s/audio_files", src_db_dir); + dst_aft_dir = make_message("%s/audio-files", src_db_dir); + PARA_NOTICE_LOG("source aft dir: %s\n", src_aft_dir); + PARA_NOTICE_LOG("destination aft dir: %s\n", dst_aft_dir); +} + +static void check_sanity(void) +{ + PARA_INFO_LOG("checking source and destination directories\n"); + if (!is_dir(src_db_dir)) + die("source db directory does not exist"); + if (path_exists(dst_db_dir)) + die("destination db already exists"); + if (!is_dir(src_aft_dir)) + die("source audio file table does not exist"); + if (path_exists(dst_aft_dir)) + die("destination audio file table already exists"); +} + +/** The columns of the audio file table (both old and new). */ +enum audio_file_table_columns { + /** The hash on the content of the audio file. */ + AFTCOL_HASH, + /** The full path in the filesystem. */ + AFTCOL_PATH, + /** The audio file selector info. */ + AFTCOL_AFSI, + /** The audio format handler info. */ + AFTCOL_AFHI, + /** The chunk table info and the chunk table of the audio file. */ + AFTCOL_CHUNKS, + /** The number of columns of this table. */ + NUM_AFT_COLUMNS +}; + +#define AFSI_SIZE 32 + +static int src_aft_hash_compare(const struct osl_object *obj1, + const struct osl_object *obj2) +{ + return hash_compare((unsigned char *)obj1->data, + (unsigned char *)obj2->data); +} + +static struct osl_column_description src_aft_cols[] = { + [AFTCOL_HASH] = { + .storage_type = OSL_MAPPED_STORAGE, + .storage_flags = OSL_RBTREE | OSL_FIXED_SIZE | OSL_UNIQUE, + .name = "hash", + .compare_function = src_aft_hash_compare, + .data_size = HASH_SIZE + }, + [AFTCOL_PATH] = { + .storage_type = OSL_MAPPED_STORAGE, + .storage_flags = OSL_RBTREE | OSL_UNIQUE, + .name = "path", + .compare_function = string_compare, + }, + [AFTCOL_AFSI] = { + .storage_type = OSL_MAPPED_STORAGE, + .storage_flags = OSL_FIXED_SIZE, + .name = "afs_info", + .data_size = AFSI_SIZE + }, + [AFTCOL_AFHI] = { + .storage_type = OSL_MAPPED_STORAGE, + .name = "afh_info", + }, + [AFTCOL_CHUNKS] = { + .storage_type = OSL_DISK_STORAGE, + .name = "chunks", + } +}; + +static struct osl_table_description src_aft_desc = { + .name = "audio_files", + .num_columns = NUM_AFT_COLUMNS, + .flags = OSL_LARGE_TABLE, + .column_descriptions = src_aft_cols +}; + +static struct osl_table *src_aft, *dst_aft; + +static void open_src_aft(void) +{ + int ret; + + PARA_NOTICE_LOG("opening: %s\n", src_aft_dir); + src_aft_desc.dir = src_db_dir; + ret = osl(osl_open_table(&src_aft_desc, &src_aft)); + if (ret < 0) { + PARA_EMERG_LOG("can not open source audio file table: %s\n", + para_strerror(-ret)); + exit(EXIT_FAILURE); + } + PARA_INFO_LOG("successfully opened source audio file table\n"); +} + +static int dst_aft_hash_compare(const struct osl_object *obj1, + const struct osl_object *obj2) +{ + return hash2_compare((unsigned char *)obj1->data, + (unsigned char *)obj2->data); +} + +/* identical to src_aft_cols except the comparator and the hash size. */ +static struct osl_column_description dst_aft_cols[] = { + [AFTCOL_HASH] = { + .storage_type = OSL_MAPPED_STORAGE, + .storage_flags = OSL_RBTREE | OSL_FIXED_SIZE | OSL_UNIQUE, + .name = "hash", + .compare_function = dst_aft_hash_compare, + .data_size = HASH2_SIZE + }, + [AFTCOL_PATH] = { + .storage_type = OSL_MAPPED_STORAGE, + .storage_flags = OSL_RBTREE | OSL_UNIQUE, + .name = "path", + .compare_function = string_compare, + }, + [AFTCOL_AFSI] = { + .storage_type = OSL_MAPPED_STORAGE, + .storage_flags = OSL_FIXED_SIZE, + .name = "afs_info", + .data_size = AFSI_SIZE + }, + [AFTCOL_AFHI] = { + .storage_type = OSL_MAPPED_STORAGE, + .name = "afh_info", + }, + [AFTCOL_CHUNKS] = { + .storage_type = OSL_DISK_STORAGE, + .name = "chunks", + } +}; + +static struct osl_table_description dst_aft_desc = { + .name = "audio-files", + .num_columns = NUM_AFT_COLUMNS, + .flags = OSL_LARGE_TABLE, + .column_descriptions = dst_aft_cols +}; + +static int create_and_open_dst_aft(void) +{ + int ret; + + PARA_NOTICE_LOG("creating %s\n", dst_aft_dir); + dst_aft_desc.dir = src_db_dir; + ret = osl(osl_create_table(&dst_aft_desc)); + if (ret < 0) { + PARA_EMERG_LOG("could not create destination audio file table\n"); + return ret; + } + ret = osl(osl_open_table(&dst_aft_desc, &dst_aft)); + if (ret < 0) { + PARA_EMERG_LOG("could not open destination audio file table: %s\n", + para_strerror(-ret)); + exit(EXIT_FAILURE); + } + PARA_INFO_LOG("successfully opened destination audio file table\n"); + return 0; +} + +static int copy_aft_row(struct osl_row *row, void *data) +{ + unsigned *n = data; + int i, ret; + unsigned char hash2[HASH2_SIZE] = "\0"; + struct osl_object objs[NUM_AFT_COLUMNS] = { + [AFTCOL_HASH] = {.data = hash2, .size = HASH2_SIZE} + }; + + ret = osl(osl_open_disk_object(src_aft, row, AFTCOL_CHUNKS, + objs + AFTCOL_CHUNKS)); + if (ret < 0) { + PARA_ERROR_LOG("can not open disk object: %s\n", + para_strerror(-ret)); + return ret; + } + for (i = 0; i < NUM_AFT_COLUMNS; i++) { + if (i == AFTCOL_HASH) /* never assign to this index */ + continue; + if (i == AFTCOL_CHUNKS) /* disk storage object handled above */ + continue; + /* mapped storage */ + ret = osl(osl_get_object(src_aft, row, i, objs + i)); + if (ret < 0) { + PARA_ERROR_LOG("get_object (col = %d): %s\n", + i, para_strerror(-ret)); + return ret; + } + if (i == AFTCOL_PATH) + PARA_DEBUG_LOG("copying %s\n", (char *)objs[i].data); + } + (*n)++; + memcpy(hash2, n, sizeof(*n)); + ret = osl(osl_add_row(dst_aft, objs)); + if (ret < 0) + PARA_ERROR_LOG("failed to add row: %s\n", para_strerror(-ret)); + osl_close_disk_object(objs + AFTCOL_CHUNKS); + return ret; +} + +static int convert_aft(void) +{ + unsigned n; + int ret; + + osl_get_num_rows(src_aft, &n); + PARA_NOTICE_LOG("converting hash of %u rows to sha256\n", n); + n = 0; + ret = osl(osl_rbtree_loop(src_aft, AFTCOL_HASH, &n, copy_aft_row)); + if (ret < 0) + PARA_ERROR_LOG("osl_rbtree_loop failed\n"); + return ret; +} + +static int remove_source_aft(void) +{ + pid_t pid; + int fds[3] = {-1, -1, -1}; /* no redirection of stdin/stdout/stderr */ + int ret, wstatus; + char *cmdline = make_message("rm -rf %s", src_aft_dir); + + PARA_NOTICE_LOG("removing %s\n", src_aft_dir); + ret = para_exec_cmdline_pid(&pid, cmdline, fds); + if (ret < 0) { + PARA_ERROR_LOG("exec failure\n"); + goto out; + } + do { + ret = waitpid(pid, &wstatus, 0); + } while (ret < 0 && errno == EINTR); + if (ret < 0) + PARA_ERROR_LOG("waitpid failure\n"); +out: + return ret; +} + +static int rename_db(void) +{ + PARA_NOTICE_LOG("renaming %s -> %s\n", src_db_dir, dst_db_dir); + if (rename(src_db_dir, dst_db_dir) < 0) { + int ret = -ERRNO_TO_PARA_ERROR(errno); + PARA_ERROR_LOG("rename failed\n"); + return ret; + } + return 1; +} + +int main(int argc, char *argv[]) +{ + struct lls_parse_result *lpr; /* command line */ + char *errctx; + int ret; + + ret = lls(lls_parse(argc, argv, CMD_PTR, &lpr, &errctx)); + if (ret < 0) + goto out; + loglevel = OPT_UINT32_VAL(LOGLEVEL, lpr); + version_handle_flag("recv", OPT_GIVEN(VERSION, lpr)); + handle_help_flag(lpr); + set_paths(lpr); + check_sanity(); + open_src_aft(); + ret = create_and_open_dst_aft(); + if (ret < 0) + goto close_src_aft; + ret = convert_aft(); + if (ret < 0) + goto close_dst_aft; + ret = remove_source_aft(); + if (ret < 0) + goto close_dst_aft; + ret = rename_db(); +close_dst_aft: + osl_close_table(dst_aft, OSL_MARK_CLEAN); +close_src_aft: + PARA_INFO_LOG("closing audio file tables\n"); + osl_close_table(src_aft, OSL_MARK_CLEAN); +out: + if (ret < 0) { + if (errctx) + PARA_ERROR_LOG("%s\n", errctx); + free(errctx); + PARA_ERROR_LOG("%s\n", para_strerror(-ret)); + } else { + PARA_WARNING_LOG("success. Now start para_server and force-add" + " all audio files.\n"); + } + return ret < 0? EXIT_FAILURE : EXIT_SUCCESS; +} -- 2.39.2