2 * Copyright (C) 2008 Andre Noll <maan@systemlinux.org>
4 * Licensed under the GPL v2. For licencing details see COPYING.
7 /** \file create.c \brief The create mode of adu. */
9 #include <dirent.h> /* readdir() */
12 #include "gcc-compat.h"
20 /* Id of the device containing the base dir. */
21 static dev_t device_id;
22 static struct bloom *global_bloom_filter;
23 static struct bloom *user_bloom_filter;
25 static int consider_bloom(struct stat64 *s)
27 if (!global_bloom_filter)
34 /** Data size to hash for the global bloom filter. */
35 #define GLOBAL_BLOOM_BUF_SIZE (sizeof(ino_t) + sizeof(dev_t) + sizeof(off_t))
36 /** For the user bloom filter also the uid is being hashed. */
37 #define USER_BLOOM_BUF_SIZE (GLOBAL_BLOOM_BUF_SIZE + sizeof(uid_t))
39 static void make_bloom_buf(struct stat64 *s, uint8_t buf[USER_BLOOM_BUF_SIZE])
43 if (!consider_bloom(s))
45 memcpy(p, &s->st_ino, sizeof(ino_t));
47 memcpy(p, &s->st_dev, sizeof(dev_t));
49 memcpy(p, &s->st_size, sizeof(off_t));
51 memcpy(p, &s->st_uid, sizeof(uid_t));
54 static int insert_global_bloom(struct stat64 *s,
55 uint8_t buf[USER_BLOOM_BUF_SIZE])
57 if (!consider_bloom(s))
59 return bloom_test_and_insert(buf, GLOBAL_BLOOM_BUF_SIZE,
63 static int insert_user_bloom(struct stat64 *s,
64 uint8_t buf[USER_BLOOM_BUF_SIZE])
66 if (!consider_bloom(s))
68 return bloom_test_and_insert(buf, USER_BLOOM_BUF_SIZE,
72 static int add_directory(char *dirname, uint64_t *dir_num, uint64_t *parent_dir_num,
73 uint64_t *dir_size, uint64_t *dir_files)
75 struct osl_object dir_objects[NUM_DT_COLUMNS];
77 INFO_LOG("adding #%llu: %s\n", (long long unsigned)*dir_num, dirname);
78 dir_objects[DT_NAME].data = dirname;
79 dir_objects[DT_NAME].size = strlen(dirname) + 1;
80 dir_objects[DT_NUM].data = dir_num;
81 dir_objects[DT_NUM].size = sizeof(*dir_num);
82 dir_objects[DT_PARENT_NUM].data = parent_dir_num;
83 dir_objects[DT_PARENT_NUM].size = sizeof(*parent_dir_num);
84 dir_objects[DT_BYTES].data = dir_size;
85 dir_objects[DT_BYTES].size = sizeof(*dir_size);
86 dir_objects[DT_FILES].data = dir_files;
87 dir_objects[DT_FILES].size = sizeof(*dir_files);
88 return osl(osl_add_row(dir_table, dir_objects));
91 static int update_user_row(struct osl_table *t, uint64_t dir_num,
95 struct osl_object obj = {.data = &dir_num, .size = sizeof(dir_num)};
97 int ret = osl(osl_get_row(t, UT_DIR_NUM, &obj, &row));
99 if (ret == -E_OSL && osl_errno != E_OSL_RB_KEY_NOT_FOUND)
101 if (ret < 0) { /* this is the first file we add */
102 struct osl_object objects[NUM_UT_COLUMNS];
103 uint64_t num_files = 1;
105 objects[UT_DIR_NUM].data = &dir_num;
106 objects[UT_DIR_NUM].size = sizeof(dir_num);
107 objects[UT_BYTES].data = &add;
108 objects[UT_BYTES].size = sizeof(add);
109 objects[UT_FILES].data = &num_files;
110 objects[UT_FILES].size = sizeof(num_files);
111 ret = osl(osl_add_row(t, objects));
113 } else { /* add size and increment file count */
115 struct osl_object obj1, obj2 = {.data = &num, .size = sizeof(num)};
117 ret = osl(osl_get_object(t, row, UT_BYTES, &obj1));
120 num = *(uint64_t *)obj1.data + add;
121 ret = osl(osl_update_object(t, row, UT_BYTES, &obj2));
124 ret = osl(osl_get_object(t, row, UT_FILES, &obj1));
127 num = *(uint64_t *)obj1.data + 1;
128 return osl(osl_update_object(t, row, UT_FILES, &obj2));
132 static int scan_dir(char *dirname, uint64_t *parent_dir_num)
135 struct dirent *entry;
136 int ret, cwd_fd, ret2;
137 uint64_t dir_size = 0, dir_files = 0;
139 static uint64_t current_dir_num;
140 uint64_t this_dir_num = ++current_dir_num;
143 DEBUG_LOG("----------------- %llu: %s\n", (long long unsigned)current_dir_num, dirname);
144 ret = adu_opendir(dirname, &dir, &cwd_fd);
146 if (ret != -ERRNO_TO_ERROR(EACCES))
148 WARNING_LOG("permission denied for %s\n", dirname);
151 while ((entry = readdir(dir))) {
154 struct user_info *ui;
155 uint8_t bloom_buf[USER_BLOOM_BUF_SIZE];
157 if (!strcmp(entry->d_name, "."))
159 if (!strcmp(entry->d_name, ".."))
161 if (lstat64(entry->d_name, &s) == -1) {
162 WARNING_LOG("lstat64 error for %s/%s (%s)\n",
163 dirname, entry->d_name, strerror(errno));
167 if (!S_ISREG(m) && !S_ISDIR(m))
170 if (conf.one_file_system_given && s.st_dev != device_id)
172 dir_size += s.st_size;
174 ret = create_user_table(s.st_uid, &ui);
177 ret = update_user_row(ui->table, this_dir_num,
181 ret = scan_dir(entry->d_name, &this_dir_num);
188 make_bloom_buf(&s, bloom_buf);
189 if (insert_global_bloom(&s, bloom_buf))
190 DEBUG_LOG("global hard link: %s/%s\n", dirname,
193 dir_size += s.st_size;
195 ret = create_user_table(s.st_uid, &ui);
198 ret = insert_user_bloom(&s, bloom_buf);
200 DEBUG_LOG("hard link for uid %d: %s/%s\n",
201 (unsigned)s.st_uid, dirname, entry->d_name);
202 ret = update_user_row(ui->table, this_dir_num,
207 ret = add_directory(dirname, &this_dir_num, parent_dir_num,
208 &dir_size, &dir_files);
211 ret2 = adu_fchdir(cwd_fd);
212 if (ret2 < 0 && ret >= 0)
218 static void log_bloom_stat(struct bloom *b)
222 NOTICE_LOG("\tfilter contains %llu entries\n",
223 (long long unsigned)b->num_entries);
224 percent = b->num_set_bits * 100ULL / (1ULL << b->order);
225 NOTICE_LOG("\t%u%% of bits are set\n", percent);
227 WARNING_LOG("results may be unreliable!\n");
228 WARNING_LOG("consider incrasing bllom filter size\n");
232 static void log_bloom_stats(void)
234 struct bloom *b = global_bloom_filter;
237 NOTICE_LOG("global bloom filter statistics:\n");
239 NOTICE_LOG("user bloom filter statistics:\n");
240 b = user_bloom_filter;
245 * The main function of the create mode.
251 uint64_t zero = 0ULL;
252 int ret, order = conf.bloom_filter_order_arg,
253 num = conf.num_bloom_filter_hash_functions_arg;
256 if (lstat(conf.base_dir_arg, &statbuf) == -1)
257 return -ERRNO_TO_ERROR(errno);
258 if (!S_ISDIR(statbuf.st_mode))
259 return -ERRNO_TO_ERROR(ENOTDIR);
260 if (order >= 10 && num > 0) {
261 bloom_init(order, num, &global_bloom_filter);
262 bloom_init(order, num, &user_bloom_filter);
264 WARNING_LOG("hard link detection deactivated\n");
265 device_id = statbuf.st_dev;
266 create_hash_table(conf.hash_table_bits_arg);
267 ret = open_dir_table(1);
271 ret = scan_dir(conf.base_dir_arg, &zero);
274 ret = write_uid_file();
277 bloom_free(global_bloom_filter);
278 bloom_free(user_bloom_filter);