2 * Copyright (C) 2008 Andre Noll <maan@systemlinux.org>
4 * Licensed under the GPL v2. For licencing details see COPYING.
7 /** \file create.c \brief The create mode of adu. */
9 #include <dirent.h> /* readdir() */
12 #include "gcc-compat.h"
20 /* Id of the device containing the base dir. */
21 static dev_t device_id;
22 static struct bloom *global_bloom_filter;
23 static struct bloom *user_bloom_filter;
25 static int consider_bloom(struct stat64 *s)
27 if (!global_bloom_filter)
34 /** Data size to hash for the global bloom filter. */
35 #define GLOBAL_BLOOM_BUF_SIZE (sizeof(ino_t) + sizeof(dev_t) + sizeof(off_t))
36 /** For the user bloom filter also the uid is being hashed. */
37 #define USER_BLOOM_BUF_SIZE (GLOBAL_BLOOM_BUF_SIZE + sizeof(uid_t))
39 static void make_bloom_buf(struct stat64 *s, uint8_t buf[USER_BLOOM_BUF_SIZE])
43 if (!consider_bloom(s))
45 memcpy(p, &s->st_ino, sizeof(ino_t));
47 memcpy(p, &s->st_dev, sizeof(dev_t));
49 memcpy(p, &s->st_size, sizeof(off_t));
51 memcpy(p, &s->st_uid, sizeof(uid_t));
54 static int insert_global_bloom(struct stat64 *s,
55 uint8_t buf[USER_BLOOM_BUF_SIZE])
57 if (!consider_bloom(s))
59 return bloom_insert(buf, GLOBAL_BLOOM_BUF_SIZE, global_bloom_filter);
62 static int insert_user_bloom(struct stat64 *s,
63 uint8_t buf[USER_BLOOM_BUF_SIZE])
65 if (!consider_bloom(s))
67 return bloom_insert(buf, USER_BLOOM_BUF_SIZE, user_bloom_filter);
70 static int add_directory(char *dirname, uint64_t *dir_num, uint64_t *parent_dir_num,
71 uint64_t *dir_size, uint64_t *dir_files)
73 struct osl_object dir_objects[NUM_DT_COLUMNS];
75 INFO_LOG("adding #%llu: %s\n", (long long unsigned)*dir_num, dirname);
76 dir_objects[DT_NAME].data = dirname;
77 dir_objects[DT_NAME].size = strlen(dirname) + 1;
78 dir_objects[DT_NUM].data = dir_num;
79 dir_objects[DT_NUM].size = sizeof(*dir_num);
80 dir_objects[DT_PARENT_NUM].data = parent_dir_num;
81 dir_objects[DT_PARENT_NUM].size = sizeof(*parent_dir_num);
82 dir_objects[DT_BYTES].data = dir_size;
83 dir_objects[DT_BYTES].size = sizeof(*dir_size);
84 dir_objects[DT_FILES].data = dir_files;
85 dir_objects[DT_FILES].size = sizeof(*dir_files);
86 return osl(osl_add_row(dir_table, dir_objects));
89 static int update_user_row(struct osl_table *t, uint64_t dir_num,
93 struct osl_object obj = {.data = &dir_num, .size = sizeof(dir_num)};
95 int ret = osl(osl_get_row(t, UT_DIR_NUM, &obj, &row));
97 if (ret == -E_OSL && osl_errno != E_OSL_RB_KEY_NOT_FOUND)
99 if (ret < 0) { /* this is the first file we add */
100 struct osl_object objects[NUM_UT_COLUMNS];
101 uint64_t num_files = 1;
103 objects[UT_DIR_NUM].data = &dir_num;
104 objects[UT_DIR_NUM].size = sizeof(dir_num);
105 objects[UT_BYTES].data = &add;
106 objects[UT_BYTES].size = sizeof(add);
107 objects[UT_FILES].data = &num_files;
108 objects[UT_FILES].size = sizeof(num_files);
109 ret = osl(osl_add_row(t, objects));
111 } else { /* add size and increment file count */
113 struct osl_object obj1, obj2 = {.data = &num, .size = sizeof(num)};
115 ret = osl(osl_get_object(t, row, UT_BYTES, &obj1));
118 num = *(uint64_t *)obj1.data + add;
119 ret = osl(osl_update_object(t, row, UT_BYTES, &obj2));
122 ret = osl(osl_get_object(t, row, UT_FILES, &obj1));
125 num = *(uint64_t *)obj1.data + 1;
126 return osl(osl_update_object(t, row, UT_FILES, &obj2));
130 static int scan_dir(char *dirname, uint64_t *parent_dir_num)
133 struct dirent *entry;
134 int ret, cwd_fd, ret2;
135 uint64_t dir_size = 0, dir_files = 0;
137 static uint64_t current_dir_num;
138 uint64_t this_dir_num = ++current_dir_num;
141 DEBUG_LOG("----------------- %llu: %s\n", (long long unsigned)current_dir_num, dirname);
142 ret = adu_opendir(dirname, &dir, &cwd_fd);
144 if (ret != -ERRNO_TO_ERROR(EACCES))
146 WARNING_LOG("permission denied for %s\n", dirname);
149 while ((entry = readdir(dir))) {
152 struct user_info *ui;
153 uint8_t bloom_buf[USER_BLOOM_BUF_SIZE];
155 if (!strcmp(entry->d_name, "."))
157 if (!strcmp(entry->d_name, ".."))
159 if (lstat64(entry->d_name, &s) == -1) {
160 WARNING_LOG("lstat64 error for %s/%s (%s)\n",
161 dirname, entry->d_name, strerror(errno));
165 if (!S_ISREG(m) && !S_ISDIR(m))
168 if (conf.one_file_system_given && s.st_dev != device_id)
170 dir_size += s.st_size;
172 ret = create_user_table(s.st_uid, &ui);
175 ret = update_user_row(ui->table, this_dir_num,
179 ret = scan_dir(entry->d_name, &this_dir_num);
186 make_bloom_buf(&s, bloom_buf);
187 if (insert_global_bloom(&s, bloom_buf))
188 DEBUG_LOG("global hard link: %s/%s\n", dirname,
191 dir_size += s.st_size;
193 ret = create_user_table(s.st_uid, &ui);
196 ret = insert_user_bloom(&s, bloom_buf);
198 DEBUG_LOG("hard link for uid %d: %s/%s\n",
199 (unsigned)s.st_uid, dirname, entry->d_name);
200 ret = update_user_row(ui->table, this_dir_num,
205 ret = add_directory(dirname, &this_dir_num, parent_dir_num,
206 &dir_size, &dir_files);
209 ret2 = adu_fchdir(cwd_fd);
210 if (ret2 < 0 && ret >= 0)
216 static void log_bloom_stat(struct bloom *b)
220 NOTICE_LOG("\tfilter contains %llu entries\n",
221 (long long unsigned)b->num_entries);
222 percent = b->num_set_bits * 100ULL / (1ULL << b->order);
223 NOTICE_LOG("\t%u%% of bits are set\n", percent);
225 WARNING_LOG("results may be unreliable!\n");
226 WARNING_LOG("consider increasing bloom filter size\n");
230 static void log_bloom_stats(void)
232 struct bloom *b = global_bloom_filter;
235 NOTICE_LOG("global bloom filter statistics:\n");
237 NOTICE_LOG("user bloom filter statistics:\n");
238 b = user_bloom_filter;
243 * The main function of the create mode.
249 uint64_t zero = 0ULL;
250 int ret, order = conf.bloom_filter_order_arg,
251 num = conf.num_bloom_filter_hash_functions_arg;
254 if (lstat(conf.base_dir_arg, &statbuf) == -1)
255 return -ERRNO_TO_ERROR(errno);
256 if (!S_ISDIR(statbuf.st_mode))
257 return -ERRNO_TO_ERROR(ENOTDIR);
258 if (order >= 10 && num > 0) {
259 global_bloom_filter = bloom_new(order, num);
260 user_bloom_filter = bloom_new(order, num);
262 WARNING_LOG("hard link detection deactivated\n");
263 device_id = statbuf.st_dev;
264 create_hash_table(conf.hash_table_bits_arg);
265 ret = open_dir_table(1);
269 ret = scan_dir(conf.base_dir_arg, &zero);
272 ret = write_uid_file();
275 bloom_free(global_bloom_filter);
276 bloom_free(user_bloom_filter);