From 1ae30c7805ddf80e6e5eb59d9332886e2a8d2640 Mon Sep 17 00:00:00 2001 From: Andre Noll Date: Sun, 25 May 2008 23:00:30 +0200 Subject: [PATCH] Implement the select command. This seperates the creation of the osl tables and the pretty-printing of the statistics. To do this, it is necessary to close the tables properly on exit. This change also fixed some memory leaks. For the select command the new code reads the uid file and recomputes the gloabal and the user stats from the osl tables. --- adu.c | 510 +++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 376 insertions(+), 134 deletions(-) diff --git a/adu.c b/adu.c index 5f674dc..8c84c82 100644 --- a/adu.c +++ b/adu.c @@ -11,6 +11,18 @@ DEFINE_ERRLIST; #define DATABASE_DIR "/tmp/adu" +#define UID_LIST DATABASE_DIR "/" "uid_list" + +struct user_info { + uint32_t uid; + struct osl_table *table; + uint64_t files; + uint64_t bytes; + uint64_t dirs; + struct osl_table_description *desc; +}; + +static struct user_info *uid_hash_table; /** evaluates to 1 if x < y, to -1 if x > y and to 0 if x == y */ #define NUM_COMPARE(x, y) ((int)((x) < (y)) - (int)((x) > (y))) @@ -76,18 +88,42 @@ static int size_compare(const struct osl_object *obj1, const struct osl_object * * are taken into account. * * \return It returns an integer less than, equal to, or greater than zero if - * \a obj1 is found, respectively, to be less than, to match, or be greater than - * obj2. + * \a obj1 is found, respectively, to be less than, to match, or be greater + * than obj2. * * \sa strcmp(3), strncmp(3), osl_compare_func. */ -int string_compare(const struct osl_object *obj1, const struct osl_object *obj2) +static int string_compare(const struct osl_object *obj1, + const struct osl_object *obj2) { const char *str1 = (const char *)obj1->data; const char *str2 = (const char *)obj2->data; return strncmp(str1, str2, MIN(obj1->size, obj2->size)); } +/** + * Compare two osl objects pointing to unsigned integers of 64 bit size. + * + * \param obj1 Pointer to the first integer. + * \param obj2 Pointer to the second integer. + * + * \return The values required for an osl compare function. + * + * \sa osl_compare_func, osl_hash_compare(). + */ +static int uint64_compare(const struct osl_object *obj1, + const struct osl_object *obj2) +{ + uint64_t d1 = read_u64((const char *)obj1->data); + uint64_t d2 = read_u64((const char *)obj2->data); + + if (d1 < d2) + return 1; + if (d1 > d2) + return -1; + return 0; +} + /** The columns of the directory table. */ enum dir_table_columns { /** The name of the directory. */ @@ -113,8 +149,8 @@ static struct osl_column_description dir_table_cols[] = { .storage_type = OSL_MAPPED_STORAGE, .storage_flags = OSL_RBTREE | OSL_FIXED_SIZE | OSL_UNIQUE, .name = "num", - .compare_function = uint32_compare, - .data_size = sizeof(uint32_t) + .compare_function = uint64_compare, + .data_size = sizeof(uint64_t) }, [DT_BYTES] = { .storage_type = OSL_MAPPED_STORAGE, @@ -157,8 +193,8 @@ static struct osl_column_description user_table_cols[] = { .storage_type = OSL_MAPPED_STORAGE, .storage_flags = OSL_RBTREE | OSL_FIXED_SIZE | OSL_UNIQUE, .name = "dir_num", - .compare_function = uint32_compare, - .data_size = sizeof(uint32_t) + .compare_function = uint64_compare, + .data_size = sizeof(uint64_t) }, [UT_BYTES] = { .storage_type = OSL_MAPPED_STORAGE, @@ -178,12 +214,12 @@ static struct osl_column_description user_table_cols[] = { static struct osl_table *dir_table; -int add_directory(char *dirname, uint32_t dir_num, uint64_t *dir_size, +static int add_directory(char *dirname, uint64_t dir_num, uint64_t *dir_size, uint64_t *dir_files) { struct osl_object dir_objects[NUM_DT_COLUMNS]; - INFO_LOG("adding #%u: %s\n", dir_num, dirname); + INFO_LOG("adding #%llu: %s\n", (long long unsigned)dir_num, dirname); dir_objects[DT_NAME].data = dirname; dir_objects[DT_NAME].size = strlen(dirname) + 1; dir_objects[DT_NUM].data = &dir_num; @@ -198,87 +234,113 @@ int add_directory(char *dirname, uint32_t dir_num, uint64_t *dir_size, static uint32_t num_uids; -int create_and_open_user_table(uint32_t uid, struct osl_table **t) +static int open_user_table(struct user_info *ui, int create) { int ret; - struct osl_table_description *desc = para_malloc(sizeof(*desc)); - desc->num_columns = NUM_UT_COLUMNS; - desc->flags = 0; - desc->column_descriptions = user_table_cols; - desc->dir = para_strdup(DATABASE_DIR); - desc->name = make_message("%u", uid); + ui->desc = para_malloc(sizeof(*ui->desc)); + ui->desc->num_columns = NUM_UT_COLUMNS; + ui->desc->flags = 0; + ui->desc->column_descriptions = user_table_cols; + ui->desc->dir = para_strdup(DATABASE_DIR); + ui->desc->name = make_message("%u", (unsigned)ui->uid); num_uids++; INFO_LOG(".............................uid #%u: %u\n", - (unsigned)num_uids, (unsigned)uid); -// user_table_desc.name = make_message("%u", uid); - ret = osl_create_table(desc); + (unsigned)num_uids, (unsigned)ui->uid); + if (create) { + ret = osl_create_table(ui->desc); + if (ret < 0) + goto err; + } + ret = osl_open_table(ui->desc, &ui->table); if (ret < 0) - return ret; - return osl_open_table(desc, t); + goto err; + return 1; +err: + free((char *)ui->desc->name); + free((char *)ui->desc->dir); + free(ui->desc); + ui->desc->name = NULL; + ui->desc->dir = NULL; + ui->desc = NULL; + ui->table = NULL; + return ret; } - #define uid_hash_bits 8 static uint32_t uid_hash_table_size = 1 << uid_hash_bits; #define PRIME1 0x811c9dc5 #define PRIME2 0x01000193 -struct user_info { - uint32_t uid; - struct osl_table *table; - uint64_t files; - uint64_t bytes; -}; - -static struct user_info *uid_hash_table; - static void create_hash_table(void) { uid_hash_table = para_calloc(uid_hash_table_size * sizeof(struct user_info)); } +static void free_hash_table(void) +{ + free(uid_hash_table); + uid_hash_table = NULL; +} + static int create_tables(void) { + int ret; + + ret = osl_create_table(&dir_table_desc); + if (ret < 0) + return ret; create_hash_table(); - return osl_create_table(&dir_table_desc); + return 1; } static uint32_t double_hash(uint32_t uid, uint32_t probe_num) { - return (uid * PRIME1 + ((uid * PRIME2) | 1) * probe_num) % uid_hash_table_size; + return (uid * PRIME1 + ((uid * PRIME2) | 1) * probe_num) + % uid_hash_table_size; } -#define FOR_EACH_USER(ui) for (ui = uid_hash_table; ui < uid_hash_table \ +#define FOR_EACH_USER(ui) for (ui = uid_hash_table; ui && ui < uid_hash_table \ + uid_hash_table_size; ui++) -static int search_uid(uint32_t uid, int insert, struct user_info **ui) +enum search_uid_flags { + OPEN_USER_TABLE = 1, + CREATE_USER_TABLE = 2, +}; + +static int search_uid(uint32_t uid, enum search_uid_flags flags, + struct user_info **ui_ptr) { uint32_t p; for (p = 0; p < uid_hash_table_size; p++) { - struct user_info *i = uid_hash_table + double_hash(uid, p); - if (!i->table) { - if (!insert) + struct user_info *ui = uid_hash_table + double_hash(uid, p); + + if (!ui->table) { + int ret; + + if (!flags) return -E_BAD_UID; - int ret = create_and_open_user_table(uid, &i->table); + ui->uid = uid; + ret = open_user_table(ui, flags & CREATE_USER_TABLE); if (ret < 0) return ret; - i->uid = uid; - *ui = i; + if (ui_ptr) + *ui_ptr = ui; return 1; } - if (i->uid != uid) + if (ui->uid != uid) continue; - *ui = i; + if (ui_ptr) + *ui_ptr = ui; return 0; } - return insert? -E_HASH_TABLE_OVERFLOW : -E_BAD_UID; + return flags? -E_HASH_TABLE_OVERFLOW : -E_BAD_UID; } -static int update_user_row(struct osl_table *t, uint32_t dir_num, +static int update_user_row(struct osl_table *t, uint64_t dir_num, uint64_t *add) { struct osl_row *row; @@ -321,8 +383,8 @@ static int update_user_row(struct osl_table *t, uint32_t dir_num, } } -static uint32_t num_dirs; -static uint32_t num_files; +static uint64_t num_dirs; +static uint64_t num_files; static uint64_t num_bytes; int scan_dir(char *dirname) @@ -331,9 +393,9 @@ int scan_dir(char *dirname) struct dirent *entry; int ret, cwd_fd, ret2; uint64_t dir_size = 0, dir_files = 0; - uint32_t this_dir_num = num_dirs++; + uint64_t this_dir_num = num_dirs++; - DEBUG_LOG("----------------- %u: %s\n", num_dirs, dirname); + DEBUG_LOG("----------------- %llu: %s\n", (long long unsigned)num_dirs, dirname); ret = para_opendir(dirname, &dir, &cwd_fd); if (ret < 0) { if (ret != -ERRNO_TO_ERROR(EACCES)) @@ -376,7 +438,7 @@ int scan_dir(char *dirname) dir_files++; num_files++; uid = s.st_uid; - ret = search_uid(uid, 1, &ui); + ret = search_uid(uid, CREATE_USER_TABLE | OPEN_USER_TABLE, &ui); if (ret < 0) goto out; ui->bytes += size; @@ -406,41 +468,63 @@ static int get_dir_name(struct osl_row *row, char **name) return 1; } -static int print_dirname_and_size(struct osl_row *row, void *data) -{ - unsigned *count = data; - struct osl_object obj; - char *name; - int ret; +enum global_stats_flags { + GSF_PRINT_DIRNAME = 1, + GSF_PRINT_BYTES = 2, + GSF_PRINT_FILES = 4, + GSF_COMPUTE_SUMMARY = 8, +}; - if ((*count)++ > 100) - return -E_LOOP_COMPLETE; - ret = get_dir_name(row, &name); - if (ret < 0) - return ret; - ret = osl_get_object(dir_table, row, DT_BYTES, &obj); - if (ret < 0) - return ret; - printf("%s\t%llu\n", name, *(long long unsigned *)obj.data); - return 1; -} +struct global_stats_info { + uint32_t count; + enum global_stats_flags flags; +}; -static int print_dirname_and_file_count(struct osl_row *row, void *data) +static int global_stats_loop_function(struct osl_row *row, void *data) { - unsigned *count = data; + struct global_stats_info *gsi = data; struct osl_object obj; - char *name; - int ret; + char *dirname; + int ret, summary = gsi->flags & GSF_COMPUTE_SUMMARY; - if ((*count)++ > 100) + if (!gsi->count && !summary) return -E_LOOP_COMPLETE; - ret = get_dir_name(row, &name); - if (ret < 0) - return ret; - ret = osl_get_object(dir_table, row, DT_FILES, &obj); - if (ret < 0) - return ret; - printf("%s\t%llu\n", name, *(long long unsigned *)obj.data); + if (gsi->count && (gsi->flags & GSF_PRINT_DIRNAME)) { + ret = get_dir_name(row, &dirname); + if (ret < 0) + return ret; + printf("%s%s", dirname, + (gsi->flags & (GSF_PRINT_FILES | GSF_PRINT_BYTES))? + "\t" : "\n" + ); + } + if (summary || (gsi->count && (gsi->flags & GSF_PRINT_FILES))) { + uint64_t files; + ret = osl_get_object(dir_table, row, DT_FILES, &obj); + if (ret < 0) + return ret; + files = *(uint64_t *)obj.data; + if (gsi->count && (gsi->flags & GSF_PRINT_FILES)) + printf("%llu%s", (long long unsigned)files, + (gsi->flags & GSF_PRINT_BYTES)? "\t" : "\n"); + if (summary) + num_files += files; + } + if (summary || (gsi->count && (gsi->flags & GSF_PRINT_BYTES))) { + uint64_t bytes; + ret = osl_get_object(dir_table, row, DT_BYTES, &obj); + if (ret < 0) + return ret; + bytes = *(uint64_t *)obj.data; + if (gsi->count && (gsi->flags & GSF_PRINT_BYTES)) + printf("%llu\n", (long long unsigned)bytes); + if (summary) { + num_bytes += bytes; + num_dirs++; + } + } + if (gsi->count) + gsi->count--; return 1; } @@ -448,88 +532,151 @@ static void print_id_stats(void) { struct user_info *ui; + printf("--------------------- user summary (uid/dirs/files/bytes):\n"); FOR_EACH_USER(ui) { if (!ui->table) continue; - printf("%u\t%llu\t%llu\n", (unsigned)ui->uid, (long long unsigned)ui->files, + printf("%u\t%llu\t%llu\t%llu\n", (unsigned)ui->uid, + (long long unsigned)ui->dirs, + (long long unsigned)ui->files, (long long unsigned)ui->bytes); } } -struct big_dir_info { - unsigned count; - struct osl_table *user_table; +enum user_stats_flags { + USF_PRINT_DIRNAME = 1, + USF_PRINT_BYTES = 2, + USF_PRINT_FILES = 4, + USF_COMPUTE_SUMMARY = 8, +}; + +struct user_stats_info { + uint32_t count; + enum user_stats_flags flags; + struct user_info *ui; }; -static int print_big_dir(struct osl_row *row, void *data) +static int user_stats_loop_function(struct osl_row *row, void *data) { - struct big_dir_info *bdi = data; - int ret; + struct user_stats_info *usi = data; struct osl_row *dir_row; - char *dirname; - uint64_t bytes; struct osl_object obj; + int ret, summary = usi->flags & GSF_COMPUTE_SUMMARY; - if (bdi->count++ > 10) + if (!usi->count && !summary) return -E_LOOP_COMPLETE; - ret = osl_get_object(bdi->user_table, row, UT_BYTES, &obj); - if (ret < 0) - return ret; - bytes = *(uint64_t *)obj.data; - ret = osl_get_object(bdi->user_table, row, UT_DIR_NUM, &obj); - if (ret < 0) - return ret; - ret = osl_get_row(dir_table, DT_NUM, &obj, &dir_row); - if (ret < 0) - return ret; - ret = osl_get_object(dir_table, dir_row, DT_NAME, &obj); - if (ret < 0) - return ret; - dirname = obj.data; - printf("%s: %llu\n", dirname, (long long unsigned)bytes); + if (usi->count && (usi->flags & USF_PRINT_DIRNAME)) { + char *dirname; + ret = osl_get_object(usi->ui->table, row, UT_DIR_NUM, &obj); + if (ret < 0) + return ret; + ret = osl_get_row(dir_table, DT_NUM, &obj, &dir_row); + if (ret < 0) + return ret; + ret = osl_get_object(dir_table, dir_row, DT_NAME, &obj); + if (ret < 0) + return ret; + dirname = obj.data; + printf("%s%s", + dirname, + (usi->flags & (USF_PRINT_FILES | USF_PRINT_BYTES))? + "\t" : "\n" + ); + } + if (summary || (usi->count && (usi->flags & USF_PRINT_FILES))) { + uint64_t files; + ret = osl_get_object(usi->ui->table, row, UT_FILES, &obj); + if (ret < 0) + return ret; + files = *(uint64_t *)obj.data; + if (usi->count && (usi->flags & USF_PRINT_FILES)) + printf("%llu%s", + (long long unsigned)files, + (usi->flags & USF_PRINT_BYTES)? "\t" : "\n" + ); + if (summary) + usi->ui->files += files; + } + if (summary || (usi->count && (usi->flags & USF_PRINT_BYTES))) { + uint64_t bytes; + ret = osl_get_object(usi->ui->table, row, UT_BYTES, &obj); + if (ret < 0) + return ret; + bytes = *(uint64_t *)obj.data; + if (usi->count && (usi->flags & USF_PRINT_BYTES)) + printf("%llu\n", (long long unsigned)bytes); + if (summary) { + usi->ui->bytes += bytes; + usi->ui->dirs++; + } + + } + if (usi->count) + usi->count--; return 1; } -static void print_id_dir_stats(void) +static void print_user_stats(void) { struct user_info *ui; FOR_EACH_USER(ui) { - struct big_dir_info bdi = {.count = 0}; + struct user_stats_info usi = { + .count = 10, + .ui = ui + }; if (!ui->table) continue; - bdi.user_table = ui->table; - printf("************************* Big dirs owned by uid %u\n", (unsigned) ui->uid); - osl_rbtree_loop_reverse(ui->table, UT_BYTES, &bdi, print_big_dir); + usi.flags = USF_PRINT_DIRNAME | USF_PRINT_BYTES | USF_COMPUTE_SUMMARY; + printf("************************************************ uid %u\n", + (unsigned) ui->uid); + if (!ui->table) + continue; + printf("----------------- Largest dirs -------------------\n"); + osl_rbtree_loop_reverse(ui->table, UT_BYTES, &usi, + user_stats_loop_function); + printf("---------- dirs containing most files ------------\n"); + usi.count = 10; + usi.flags = USF_PRINT_DIRNAME | USF_PRINT_FILES; + osl_rbtree_loop_reverse(ui->table, UT_FILES, &usi, + user_stats_loop_function); } } static int print_statistics(void) { - unsigned count = 0; int ret; - - printf("Summary: %u dirs, %u files, %llu bytes\n", (unsigned)num_dirs, - (unsigned)num_files, (long long unsigned)num_bytes); - printf("************************* Biggest dirs\n"); - ret = osl_rbtree_loop_reverse(dir_table, DT_BYTES, &count, print_dirname_and_size); + struct global_stats_info gsi = { + .count = 10, + .flags = GSF_PRINT_DIRNAME | GSF_PRINT_BYTES | GSF_COMPUTE_SUMMARY + }; + + printf("----------------- Largest dirs -------------------\n"); + ret = osl_rbtree_loop_reverse(dir_table, DT_BYTES, &gsi, + global_stats_loop_function); if (ret < 0 && ret != -E_LOOP_COMPLETE) return ret; - count = 0; - printf("************************* dirs containing many files\n"); - ret = osl_rbtree_loop_reverse(dir_table, DT_FILES, &count, print_dirname_and_file_count); + gsi.count = 10; + + gsi.flags = GSF_PRINT_DIRNAME | GSF_PRINT_FILES; + printf("---------- dirs containing most files ------------\n"); + ret = osl_rbtree_loop_reverse(dir_table, DT_FILES, &gsi, + global_stats_loop_function); if (ret < 0 && ret != -E_LOOP_COMPLETE) return ret; - printf("************************* dirs stats by owner\n"); + printf("------------------ Global summary (dirs/files/bytes)\n" + "%llu\t%llu\t%llu\n", + (long long unsigned)num_dirs, (long long unsigned)num_files, + (long long unsigned)num_bytes); + print_user_stats(); print_id_stats(); - print_id_dir_stats(); return 1; } static int write_uid_list(void) { - char *buf, *filename = DATABASE_DIR "/" "uid_list"; + char *buf; uint32_t count = 0; struct user_info *ui; size_t size = num_uids * sizeof(uint32_t); @@ -543,35 +690,131 @@ static int write_uid_list(void) continue; write_u32(buf + count++ * sizeof(uint32_t), ui->uid); } - ret = para_write_file(filename, buf, size); + ret = para_write_file(UID_LIST, buf, size); free(buf); return ret; } -static int com_create(char *dirname) +static int open_dir_table(void) { - int ret = scan_dir(dirname); + return osl_open_table(&dir_table_desc, &dir_table); +} +static void close_dir_table(void) +{ + int ret; + + if (!dir_table) + return; + ret = osl_close_table(dir_table, OSL_MARK_CLEAN); if (ret < 0) - return ret; - return write_uid_list(); + ERROR_LOG("failed to close dir table: %s\n", error_txt(-ret)); + dir_table = NULL; } -int main(int argc, char **argv) +static void close_user_table(struct user_info *ui) +{ + int ret; + + if (!ui || !ui->table) + return; + ret = osl_close_table(ui->table, OSL_MARK_CLEAN); + if (ret < 0) + ERROR_LOG("failed to close user table %u: %s\n", + (unsigned) ui->uid, error_txt(-ret)); + free((char *)ui->desc->name); + ui->desc->name = NULL; + free((char *)ui->desc->dir); + ui->desc->dir = NULL; + free(ui->desc); + ui->desc = NULL; + ui->table = NULL; +} + +static void close_user_tables(void) +{ + struct user_info *ui; + + FOR_EACH_USER(ui) + close_user_table(ui); +} + +static void close_all_tables(void) +{ + close_dir_table(); + close_user_tables(); + free_hash_table(); +} + +static int com_create(char *dirname) { int ret = create_tables(); if (ret < 0) - goto out; - ret = osl_open_table(&dir_table_desc, &dir_table); + return ret; + ret = open_dir_table(); + if (ret < 0) + return ret; + ret = scan_dir(dirname); if (ret < 0) goto out; - ret = -E_SYNTAX; - if (argc != 2) + ret = write_uid_list(); +out: + close_all_tables(); + return ret; +} + +static int read_uid_file(void) +{ + char *map; + size_t size; + int ret = mmap_full_file(UID_LIST, O_RDONLY, (void **)&map, &size, NULL); + uint32_t n; + + if (ret < 0) + return ret; + num_uids = size / 4; + /* hash table size should be a power of two and larger than the number of uids */ + uid_hash_table_size = 4; + while (uid_hash_table_size < num_uids) + uid_hash_table_size *= 2; + create_hash_table(); + for (n = 0; n < num_uids; n++) { + uint32_t uid = read_u32(map + n * sizeof(uid)); + ret = search_uid(uid, OPEN_USER_TABLE, NULL); + if (ret < 0) + goto out; + } +out: + para_munmap(map, size); + return ret; +} + +static int com_select(void) +{ + int ret; + + ret = open_dir_table(); + if (ret < 0) + return ret; + ret = read_uid_file(); + if (ret < 0) + return ret; + print_statistics(); + close_all_tables(); + return 1; +} + +int main(int argc, char **argv) +{ + int ret = -E_SYNTAX; + if (argc > 2) goto out; - ret = com_create(argv[1]); + if (argc == 1) + ret = com_select(); + else + ret = com_create(argv[1]); if (ret < 0) goto out; - print_statistics(); out: if (ret < 0) { ERROR_LOG("%s\n", error_txt(-ret)); @@ -579,4 +822,3 @@ out: } return EXIT_SUCCESS; } - -- 2.39.2