create.c

   1 /*
   2  * Copyright (C) 2008 Andre Noll <maan@systemlinux.org>
   3  *
   4  * Licensed under the GPL v2. For licencing details see COPYING.
   5  */
   6
   7 /** \file create.c \brief The create mode of adu. */
   8
   9 #include <dirent.h> /* readdir() */
  10 #include "format.h"
  11 #include "adu.h"
  12 #include "gcc-compat.h"
  13 #include "cmdline.h"
  14 #include "fd.h"
  15 #include "string.h"
  16 #include "error.h"
  17 #include "user.h"
  18 #include "bloom.h"
  19
  20 /* Id of the device containing the base dir. */
  21 static dev_t device_id;
  22 static struct bloom *global_bloom_filter;
  23 static struct bloom *user_bloom_filter;
  24
  25 static int consider_bloom(struct stat64 *s)
  26 {
  27         if (!global_bloom_filter)
  28                 return 0;
  29         if (s->st_nlink <= 1)
  30                 return 0;
  31         return 1;
  32 }
  33
  34 #define GLOBAL_BLOOM_BUF_SIZE (sizeof(ino_t) + sizeof(dev_t) + sizeof(off_t))
  35 #define USER_BLOOM_BUF_SIZE (GLOBAL_BLOOM_BUF_SIZE + sizeof(uid_t))
  36
  37 static void make_bloom_buf(struct stat64 *s, uint8_t buf[USER_BLOOM_BUF_SIZE])
  38 {
  39         uint8_t *p = buf;
  40
  41         if (!consider_bloom(s))
  42                 return;
  43         memcpy(p, &s->st_ino, sizeof(ino_t));
  44         p += sizeof(ino_t);
  45         memcpy(p, &s->st_dev, sizeof(dev_t));
  46         p += sizeof(dev_t);
  47         memcpy(p, &s->st_size, sizeof(off_t));
  48         p += sizeof(off_t);
  49         memcpy(p, &s->st_uid, sizeof(uid_t));
  50 }
  51
  52 static int insert_global_bloom(struct stat64 *s,
  53                 uint8_t buf[USER_BLOOM_BUF_SIZE])
  54 {
  55         if (!consider_bloom(s))
  56                 return 0;
  57         return bloom_test_and_insert(buf, GLOBAL_BLOOM_BUF_SIZE,
  58                 global_bloom_filter);
  59 }
  60
  61 static int insert_user_bloom(struct stat64 *s,
  62                 uint8_t buf[USER_BLOOM_BUF_SIZE])
  63 {
  64         if (!consider_bloom(s))
  65                 return 0;
  66         return bloom_test_and_insert(buf, USER_BLOOM_BUF_SIZE,
  67                 user_bloom_filter);
  68 }
  69
  70 static int add_directory(char *dirname, uint64_t *dir_num, uint64_t *parent_dir_num,
  71                 uint64_t *dir_size, uint64_t *dir_files)
  72 {
  73         struct osl_object dir_objects[NUM_DT_COLUMNS];
  74
  75         INFO_LOG("adding #%llu: %s\n", (long long unsigned)*dir_num, dirname);
  76         dir_objects[DT_NAME].data = dirname;
  77         dir_objects[DT_NAME].size = strlen(dirname) + 1;
  78         dir_objects[DT_NUM].data = dir_num;
  79         dir_objects[DT_NUM].size = sizeof(*dir_num);
  80         dir_objects[DT_PARENT_NUM].data = parent_dir_num;
  81         dir_objects[DT_PARENT_NUM].size = sizeof(*parent_dir_num);
  82         dir_objects[DT_BYTES].data = dir_size;
  83         dir_objects[DT_BYTES].size = sizeof(*dir_size);
  84         dir_objects[DT_FILES].data = dir_files;
  85         dir_objects[DT_FILES].size = sizeof(*dir_files);
  86         return osl(osl_add_row(dir_table, dir_objects));
  87 }
  88
  89 static int update_user_row(struct osl_table *t, uint64_t dir_num,
  90                 uint64_t add)
  91 {
  92         struct osl_row *row;
  93         struct osl_object obj = {.data = &dir_num, .size = sizeof(dir_num)};
  94
  95         int ret = osl(osl_get_row(t, UT_DIR_NUM, &obj, &row));
  96
  97         if (ret == -E_OSL && osl_errno != E_OSL_RB_KEY_NOT_FOUND)
  98                 return ret;
  99         if (ret < 0) { /* this is the first file we add */
 100                 struct osl_object objects[NUM_UT_COLUMNS];
 101                 uint64_t num_files = 1;
 102
 103                 objects[UT_DIR_NUM].data = &dir_num;
 104                 objects[UT_DIR_NUM].size = sizeof(dir_num);
 105                 objects[UT_BYTES].data = &add;
 106                 objects[UT_BYTES].size = sizeof(add);
 107                 objects[UT_FILES].data = &num_files;
 108                 objects[UT_FILES].size = sizeof(num_files);
 109                 ret = osl(osl_add_row(t, objects));
 110                 return ret;
 111         } else { /* add size and increment file count */
 112                 uint64_t num;
 113                 struct osl_object obj1, obj2 = {.data = &num, .size = sizeof(num)};
 114
 115                 ret = osl(osl_get_object(t, row, UT_BYTES, &obj1));
 116                 if (ret < 0)
 117                         return ret;
 118                 num = *(uint64_t *)obj1.data + add;
 119                 ret = osl(osl_update_object(t, row, UT_BYTES, &obj2));
 120                 if (ret < 0)
 121                         return ret;
 122                 ret = osl(osl_get_object(t, row, UT_FILES, &obj1));
 123                 if (ret < 0)
 124                         return ret;
 125                 num = *(uint64_t *)obj1.data + 1;
 126                 return osl(osl_update_object(t, row, UT_FILES, &obj2));
 127         }
 128 }
 129
 130 static int scan_dir(char *dirname, uint64_t *parent_dir_num)
 131 {
 132         DIR *dir;
 133         struct dirent *entry;
 134         int ret, cwd_fd, ret2;
 135         uint64_t dir_size = 0, dir_files = 0;
 136         /* dir count. */
 137         static uint64_t current_dir_num;
 138         uint64_t this_dir_num = ++current_dir_num;
 139
 140         check_signals();
 141         DEBUG_LOG("----------------- %llu: %s\n", (long long unsigned)current_dir_num, dirname);
 142         ret = adu_opendir(dirname, &dir, &cwd_fd);
 143         if (ret < 0) {
 144                 if (ret != -ERRNO_TO_ERROR(EACCES))
 145                         return ret;
 146                 WARNING_LOG("permission denied for %s\n", dirname);
 147                 return 1;
 148         }
 149         while ((entry = readdir(dir))) {
 150                 mode_t m;
 151                 struct stat64 s;
 152                 struct user_info *ui;
 153                 uint8_t bloom_buf[USER_BLOOM_BUF_SIZE];
 154
 155                 if (!strcmp(entry->d_name, "."))
 156                         continue;
 157                 if (!strcmp(entry->d_name, ".."))
 158                         continue;
 159                 if (lstat64(entry->d_name, &s) == -1) {
 160                         WARNING_LOG("lstat64 error for %s/%s (%s)\n",
 161                                 dirname, entry->d_name, strerror(errno));
 162                         continue;
 163                 }
 164                 m = s.st_mode;
 165                 if (!S_ISREG(m) && !S_ISDIR(m))
 166                         continue;
 167                 if (S_ISDIR(m)) {
 168                         if (conf.one_file_system_given && s.st_dev != device_id)
 169                                 continue;
 170                         dir_size += s.st_size;
 171                         dir_files++;
 172                         ret = create_user_table(s.st_uid, &ui);
 173                         if (ret < 0)
 174                                 goto out;
 175                         ret = update_user_row(ui->table, this_dir_num,
 176                                 s.st_size);
 177                         if (ret < 0)
 178                                 goto out;
 179                         ret = scan_dir(entry->d_name, &this_dir_num);
 180                         if (ret < 0)
 181                                 goto out;
 182                         continue;
 183                 }
 184
 185                 /* regular file */
 186                 make_bloom_buf(&s, bloom_buf);
 187                 if (insert_global_bloom(&s, bloom_buf))
 188                         DEBUG_LOG("global hard link: %s/%s\n", dirname,
 189                                 entry->d_name);
 190                 else
 191                         dir_size += s.st_size;
 192                 dir_files++;
 193                 ret = create_user_table(s.st_uid, &ui);
 194                 if (ret < 0)
 195                         goto out;
 196                 ret = insert_user_bloom(&s, bloom_buf);
 197                 if (ret)
 198                         DEBUG_LOG("hard link for uid %d: %s/%s\n",
 199                                 (unsigned)s.st_uid, dirname, entry->d_name);
 200                 ret = update_user_row(ui->table, this_dir_num,
 201                         ret? 0 : s.st_size);
 202                 if (ret < 0)
 203                         goto out;
 204         }
 205         ret = add_directory(dirname, &this_dir_num, parent_dir_num,
 206                         &dir_size, &dir_files);
 207 out:
 208         closedir(dir);
 209         ret2 = adu_fchdir(cwd_fd);
 210         if (ret2 < 0 && ret >= 0)
 211                 ret = ret2;
 212         close(cwd_fd);
 213         return ret;
 214 }
 215
 216 static void log_bloom_stat(struct bloom *b)
 217 {
 218         unsigned percent;
 219
 220         NOTICE_LOG("\tfilter contains %llu entries\n",
 221                 (long long unsigned)b->num_entries);
 222         percent = b->num_set_bits * 100ULL / (1ULL << b->order);
 223         NOTICE_LOG("\t%u%% of bits are set\n", percent);
 224         if (percent > 50) {
 225                 WARNING_LOG("results may be unreliable!\n");
 226                 WARNING_LOG("consider incrasing bllom filter size\n");
 227         }
 228 }
 229
 230 static void log_bloom_stats(void)
 231 {
 232         struct bloom *b = global_bloom_filter;
 233         if (!b)
 234                 return;
 235         NOTICE_LOG("global bloom filter statistics:\n");
 236         log_bloom_stat(b);
 237         NOTICE_LOG("user bloom filter statistics:\n");
 238         b = user_bloom_filter;
 239         log_bloom_stat(b);
 240 }
 241
 242 /**
 243  * The main function of the create mode.
 244  *
 245  * \return Standard.
 246  */
 247 int com_create(void)
 248 {
 249         uint64_t zero = 0ULL;
 250         int ret, order = conf.bloom_filter_order_arg,
 251                 num = conf.num_bloom_filter_hash_functions_arg;
 252         struct stat statbuf;
 253
 254         if (lstat(conf.base_dir_arg, &statbuf) == -1)
 255                 return -ERRNO_TO_ERROR(errno);
 256         if (!S_ISDIR(statbuf.st_mode))
 257                 return -ERRNO_TO_ERROR(ENOTDIR);
 258         if (order >= 10 && num > 0) {
 259                 bloom_init(order, num, &global_bloom_filter);
 260                 bloom_init(order, num, &user_bloom_filter);
 261         } else
 262                 WARNING_LOG("hard link detection deactivated\n");
 263         device_id = statbuf.st_dev;
 264         create_hash_table(conf.hash_table_bits_arg);
 265         ret = open_dir_table(1);
 266         if (ret < 0)
 267                 goto out;
 268         check_signals();
 269         ret = scan_dir(conf.base_dir_arg, &zero);
 270         if (ret < 0)
 271                 goto out;
 272         ret = write_uid_file();
 273         log_bloom_stats();
 274 out:
 275         bloom_free(global_bloom_filter);
 276         bloom_free(user_bloom_filter);
 277         return ret;
 278 }