Move adu_strerror() to adu.c.
[adu.git] / create.c
1 /*
2 * Copyright (C) 2008 Andre Noll <maan@systemlinux.org>
3 *
4 * Licensed under the GPL v2. For licencing details see COPYING.
5 */
6
7 /** \file create.c \brief The create mode of adu. */
8
9 #include <dirent.h> /* readdir() */
10 #include "format.h"
11 #include "adu.h"
12 #include "gcc-compat.h"
13 #include "cmdline.h"
14 #include "fd.h"
15 #include "string.h"
16 #include "error.h"
17 #include "user.h"
18 #include "bloom.h"
19
20 /* Id of the device containing the base dir. */
21 static dev_t device_id;
22 static struct bloom *global_bloom_filter;
23 static struct bloom *user_bloom_filter;
24
25 static int consider_bloom(struct stat64 *s)
26 {
27 if (!global_bloom_filter)
28 return 0;
29 if (s->st_nlink <= 1)
30 return 0;
31 return 1;
32 }
33
34 /** Data size to hash for the global bloom filter. */
35 #define GLOBAL_BLOOM_BUF_SIZE (sizeof(ino_t) + sizeof(dev_t) + sizeof(off_t))
36 /** For the user bloom filter also the uid is being hashed. */
37 #define USER_BLOOM_BUF_SIZE (GLOBAL_BLOOM_BUF_SIZE + sizeof(uid_t))
38
39 static void make_bloom_buf(struct stat64 *s, uint8_t buf[USER_BLOOM_BUF_SIZE])
40 {
41 uint8_t *p = buf;
42
43 if (!consider_bloom(s))
44 return;
45 memcpy(p, &s->st_ino, sizeof(ino_t));
46 p += sizeof(ino_t);
47 memcpy(p, &s->st_dev, sizeof(dev_t));
48 p += sizeof(dev_t);
49 memcpy(p, &s->st_size, sizeof(off_t));
50 p += sizeof(off_t);
51 memcpy(p, &s->st_uid, sizeof(uid_t));
52 }
53
54 static int insert_global_bloom(struct stat64 *s,
55 uint8_t buf[USER_BLOOM_BUF_SIZE])
56 {
57 if (!consider_bloom(s))
58 return 0;
59 return bloom_insert(buf, GLOBAL_BLOOM_BUF_SIZE, global_bloom_filter);
60 }
61
62 static int insert_user_bloom(struct stat64 *s,
63 uint8_t buf[USER_BLOOM_BUF_SIZE])
64 {
65 if (!consider_bloom(s))
66 return 0;
67 return bloom_insert(buf, USER_BLOOM_BUF_SIZE, user_bloom_filter);
68 }
69
70 static int add_directory(char *dirname, uint64_t *dir_num, uint64_t *parent_dir_num,
71 uint64_t *dir_size, uint64_t *dir_files)
72 {
73 struct osl_object dir_objects[NUM_DT_COLUMNS];
74
75 INFO_LOG("adding #%llu: %s\n", (long long unsigned)*dir_num, dirname);
76 dir_objects[DT_NAME].data = dirname;
77 dir_objects[DT_NAME].size = strlen(dirname) + 1;
78 dir_objects[DT_NUM].data = dir_num;
79 dir_objects[DT_NUM].size = sizeof(*dir_num);
80 dir_objects[DT_PARENT_NUM].data = parent_dir_num;
81 dir_objects[DT_PARENT_NUM].size = sizeof(*parent_dir_num);
82 dir_objects[DT_BYTES].data = dir_size;
83 dir_objects[DT_BYTES].size = sizeof(*dir_size);
84 dir_objects[DT_FILES].data = dir_files;
85 dir_objects[DT_FILES].size = sizeof(*dir_files);
86 return osl(osl_add_row(dir_table, dir_objects));
87 }
88
89 static int update_user_row(struct osl_table *t, uint64_t dir_num,
90 uint64_t add)
91 {
92 struct osl_row *row;
93 struct osl_object obj = {.data = &dir_num, .size = sizeof(dir_num)};
94
95 int ret = osl(osl_get_row(t, UT_DIR_NUM, &obj, &row));
96
97 if (ret == -E_OSL && osl_errno != E_OSL_RB_KEY_NOT_FOUND)
98 return ret;
99 if (ret < 0) { /* this is the first file we add */
100 struct osl_object objects[NUM_UT_COLUMNS];
101 uint64_t num_files = 1;
102
103 objects[UT_DIR_NUM].data = &dir_num;
104 objects[UT_DIR_NUM].size = sizeof(dir_num);
105 objects[UT_BYTES].data = &add;
106 objects[UT_BYTES].size = sizeof(add);
107 objects[UT_FILES].data = &num_files;
108 objects[UT_FILES].size = sizeof(num_files);
109 ret = osl(osl_add_row(t, objects));
110 return ret;
111 } else { /* add size and increment file count */
112 uint64_t num;
113 struct osl_object obj1, obj2 = {.data = &num, .size = sizeof(num)};
114
115 ret = osl(osl_get_object(t, row, UT_BYTES, &obj1));
116 if (ret < 0)
117 return ret;
118 num = *(uint64_t *)obj1.data + add;
119 ret = osl(osl_update_object(t, row, UT_BYTES, &obj2));
120 if (ret < 0)
121 return ret;
122 ret = osl(osl_get_object(t, row, UT_FILES, &obj1));
123 if (ret < 0)
124 return ret;
125 num = *(uint64_t *)obj1.data + 1;
126 return osl(osl_update_object(t, row, UT_FILES, &obj2));
127 }
128 }
129
130 static int scan_dir(char *dirname, uint64_t *parent_dir_num)
131 {
132 DIR *dir;
133 struct dirent *entry;
134 int ret, cwd_fd, ret2;
135 uint64_t dir_size = 0, dir_files = 0;
136 /* dir count. */
137 static uint64_t current_dir_num;
138 uint64_t this_dir_num = ++current_dir_num;
139
140 check_signals();
141 DEBUG_LOG("----------------- %llu: %s\n", (long long unsigned)current_dir_num, dirname);
142 ret = adu_opendir(dirname, &dir, &cwd_fd);
143 if (ret < 0) {
144 if (ret != -ERRNO_TO_ERROR(EACCES))
145 return ret;
146 WARNING_LOG("permission denied for %s\n", dirname);
147 return 1;
148 }
149 while ((entry = readdir(dir))) {
150 mode_t m;
151 struct stat64 s;
152 struct user_info *ui;
153 uint8_t bloom_buf[USER_BLOOM_BUF_SIZE];
154
155 if (!strcmp(entry->d_name, "."))
156 continue;
157 if (!strcmp(entry->d_name, ".."))
158 continue;
159 if (lstat64(entry->d_name, &s) == -1) {
160 WARNING_LOG("lstat64 error for %s/%s (%s)\n",
161 dirname, entry->d_name, strerror(errno));
162 continue;
163 }
164 m = s.st_mode;
165 if (!S_ISREG(m) && !S_ISDIR(m))
166 continue;
167 if (S_ISDIR(m)) {
168 if (conf.one_file_system_given && s.st_dev != device_id)
169 continue;
170 dir_size += s.st_size;
171 dir_files++;
172 ret = create_user_table(s.st_uid, &ui);
173 if (ret < 0)
174 goto out;
175 ret = update_user_row(ui->table, this_dir_num,
176 s.st_size);
177 if (ret < 0)
178 goto out;
179 ret = scan_dir(entry->d_name, &this_dir_num);
180 if (ret < 0)
181 goto out;
182 continue;
183 }
184
185 /* regular file */
186 make_bloom_buf(&s, bloom_buf);
187 if (insert_global_bloom(&s, bloom_buf))
188 DEBUG_LOG("global hard link: %s/%s\n", dirname,
189 entry->d_name);
190 else
191 dir_size += s.st_size;
192 dir_files++;
193 ret = create_user_table(s.st_uid, &ui);
194 if (ret < 0)
195 goto out;
196 ret = insert_user_bloom(&s, bloom_buf);
197 if (ret)
198 DEBUG_LOG("hard link for uid %d: %s/%s\n",
199 (unsigned)s.st_uid, dirname, entry->d_name);
200 ret = update_user_row(ui->table, this_dir_num,
201 ret? 0 : s.st_size);
202 if (ret < 0)
203 goto out;
204 }
205 ret = add_directory(dirname, &this_dir_num, parent_dir_num,
206 &dir_size, &dir_files);
207 out:
208 closedir(dir);
209 ret2 = adu_fchdir(cwd_fd);
210 if (ret2 < 0 && ret >= 0)
211 ret = ret2;
212 close(cwd_fd);
213 return ret;
214 }
215
216 static void log_bloom_stat(struct bloom *b)
217 {
218 unsigned percent;
219
220 NOTICE_LOG("\tfilter contains %llu entries\n",
221 (long long unsigned)b->num_entries);
222 percent = b->num_set_bits * 100ULL / (1ULL << b->order);
223 NOTICE_LOG("\t%u%% of bits are set\n", percent);
224 if (percent > 50) {
225 WARNING_LOG("results may be unreliable!\n");
226 WARNING_LOG("consider increasing bloom filter size\n");
227 }
228 }
229
230 static void log_bloom_stats(void)
231 {
232 struct bloom *b = global_bloom_filter;
233 if (!b)
234 return;
235 NOTICE_LOG("global bloom filter statistics:\n");
236 log_bloom_stat(b);
237 NOTICE_LOG("user bloom filter statistics:\n");
238 b = user_bloom_filter;
239 log_bloom_stat(b);
240 }
241
242 /**
243 * The main function of the create mode.
244 *
245 * \return Standard.
246 */
247 int com_create(void)
248 {
249 uint64_t zero = 0ULL;
250 int ret, order = conf.bloom_filter_order_arg,
251 num = conf.num_bloom_filter_hash_functions_arg;
252 struct stat statbuf;
253
254 if (lstat(conf.base_dir_arg, &statbuf) == -1)
255 return -ERRNO_TO_ERROR(errno);
256 if (!S_ISDIR(statbuf.st_mode))
257 return -ERRNO_TO_ERROR(ENOTDIR);
258 if (order >= 10 && num > 0) {
259 global_bloom_filter = bloom_new(order, num);
260 user_bloom_filter = bloom_new(order, num);
261 } else
262 WARNING_LOG("hard link detection deactivated\n");
263 device_id = statbuf.st_dev;
264 create_hash_table(conf.hash_table_bits_arg);
265 ret = open_dir_table(1);
266 if (ret < 0)
267 goto out;
268 check_signals();
269 ret = scan_dir(conf.base_dir_arg, &zero);
270 if (ret < 0)
271 goto out;
272 ret = write_uid_file();
273 log_bloom_stats();
274 out:
275 bloom_free(global_bloom_filter);
276 bloom_free(user_bloom_filter);
277 return ret;
278 }