adu.c

   1 #include "adu.h"
   2 #include <dirent.h> /* readdir() */
   3
   4 #include "gcc-compat.h"
   5 #include "cmdline.h"
   6 #include "fd.h"
   7 #include "string.h"
   8 #include "error.h"
   9 #include "portable_io.h"
  10
  11 DEFINE_ERRLIST;
  12
  13 /** Command line and config file options. */
  14 static struct gengetopt_args_info conf;
  15
  16 enum uid_info_flags {
  17         /** whether this slot of the hash table is used. */
  18         UI_FL_SLOT_USED = 1,
  19         /** whether this uid should be taken into account. */
  20         UI_FL_ADMISSIBLE = 2,
  21 };
  22
  23 struct user_info {
  24         uint32_t uid;
  25         uint32_t flags;
  26         struct osl_table *table;
  27         uint64_t files;
  28         uint64_t bytes;
  29         uint64_t dirs;
  30         struct osl_table_description *desc;
  31 };
  32
  33 /**
  34  * Contains info for each user that owns at least one regular file.
  35  *
  36  * Even users that are not taken into account because of the --uid
  37  * option occupy a slot in this hash table. This allows to find out
  38  * quicky whether a uid is admissible. And yes, this has to be fast.
  39  */
  40 static struct user_info *uid_hash_table;
  41
  42 static inline int ui_used(struct user_info *ui)
  43 {
  44         return ui->flags & UI_FL_SLOT_USED;
  45 }
  46
  47 static inline int ui_admissible(struct user_info *ui)
  48 {
  49         return ui->flags & UI_FL_ADMISSIBLE;
  50 }
  51
  52 struct uid_range {
  53         uint32_t low;
  54         uint32_t high;
  55 };
  56
  57 static struct uid_range *admissible_uids;
  58
  59 static inline int check_uid_arg(const char *arg, uint32_t *uid)
  60 {
  61         const uint32_t max = ~0U;
  62         /*
  63          * we need an 64-bit int for string -> uid conversion because strtoll()
  64          * returns a signed value.
  65          */
  66         int64_t val;
  67         int ret = para_atoi64(arg, &val);
  68
  69         if (ret < 0)
  70                 return ret;
  71         if (val < 0 || val > max)
  72                 return -ERRNO_TO_ERROR(EINVAL);
  73         *uid = val;
  74         return 1;
  75 }
  76
  77 static int parse_uid_range(const char *orig_arg, struct uid_range *ur)
  78 {
  79         int ret;
  80         char *arg = para_strdup(orig_arg), *p = strchr(arg, '-');
  81
  82         if (!p || p == arg) {
  83                 if (p == arg) /* -42 */
  84                         p++;
  85                 ret = check_uid_arg(p, &ur->high);
  86                 if (ret < 0)
  87                         goto out;
  88                 ur->low = p? 0 : ur->high;
  89                 ret = 1;
  90                 goto out;
  91         }
  92         /* 42- or 42-4711 */
  93         *p = '\0';
  94         p++;
  95         ret = check_uid_arg(arg, &ur->low);
  96         if (ret < 0)
  97                 goto out;
  98         ur->high = ~0U;
  99         if (*p) { /* 42-4711 */
 100                 ret = check_uid_arg(p, &ur->high);
 101                 if (ret < 0)
 102                         goto out;
 103         }
 104         if (ur->low > ur->high)
 105                 ret = -ERRNO_TO_ERROR(EINVAL);
 106 out:
 107         if (ret < 0)
 108                 ERROR_LOG("bad uid option: %s\n", orig_arg);
 109         else
 110                 INFO_LOG("admissible uid range: %u - %u\n", ur->low,
 111                         ur->high);
 112         free(arg);
 113         return ret;
 114 }
 115
 116
 117 /** evaluates to 1 if x < y, to -1 if x > y and to 0 if x == y */
 118 #define NUM_COMPARE(x, y) ((int)((x) < (y)) - (int)((x) > (y)))
 119
 120 /**
 121  * The log function.
 122  *
 123  * \param ll Loglevel.
 124  * \param fml Usual format string.
 125  *
 126  * All XXX_LOG() macros use this function.
 127  */
 128 __printf_2_3 void __log(int ll, const char* fmt,...)
 129 {
 130         va_list argp;
 131         FILE *outfd;
 132         struct tm *tm;
 133         time_t t1;
 134         char str[255] = "";
 135
 136         if (ll < conf.loglevel_arg)
 137                 return;
 138         outfd = stderr;
 139         time(&t1);
 140         tm = localtime(&t1);
 141         strftime(str, sizeof(str), "%b %d %H:%M:%S", tm);
 142         fprintf(outfd, "%s ", str);
 143         va_start(argp, fmt);
 144         vfprintf(outfd, fmt, argp);
 145         va_end(argp);
 146 }
 147
 148 /**
 149  * Compare the size of two directories
 150  *
 151  * \param obj1 Pointer to the first object.
 152  * \param obj2 Pointer to the second object.
 153  *
 154  * This function first compares the size values as usual integers. If they compare as
 155  * equal, the address of \a obj1 and \a obj2 are compared. So this compare function
 156  * returns zero if and only if \a obj1 and \a obj2 point to the same memory area.
 157  */
 158 static int size_compare(const struct osl_object *obj1, const struct osl_object *obj2)
 159 {
 160         uint64_t d1 = *(uint64_t *)obj1->data;
 161         uint64_t d2 = *(uint64_t *)obj2->data;
 162         int ret = NUM_COMPARE(d2, d1);
 163
 164         if (ret)
 165                 return ret;
 166         //INFO_LOG("addresses: %p, %p\n", obj1->data, obj2->data);
 167         return NUM_COMPARE(obj2->data, obj1->data);
 168 }
 169
 170 /**
 171  * Compare two osl objects pointing to unsigned integers of 64 bit size.
 172  *
 173  * \param obj1 Pointer to the first integer.
 174  * \param obj2 Pointer to the second integer.
 175  *
 176  * \return The values required for an osl compare function.
 177  *
 178  * \sa osl_compare_func, osl_hash_compare().
 179  */
 180 static int uint64_compare(const struct osl_object *obj1,
 181                 const struct osl_object *obj2)
 182 {
 183         uint64_t d1 = read_u64((const char *)obj1->data);
 184         uint64_t d2 = read_u64((const char *)obj2->data);
 185
 186         if (d1 < d2)
 187                 return 1;
 188         if (d1 > d2)
 189                 return -1;
 190         return 0;
 191 }
 192
 193 /** The columns of the directory table. */
 194 enum dir_table_columns {
 195         /** The name of the directory. */
 196         DT_NAME,
 197         /** The dir count number. */
 198         DT_NUM,
 199         /** The number of bytes of all regular files. */
 200         DT_BYTES,
 201         /** The number of all regular files. */
 202         DT_FILES,
 203         /** Number of columns in this table. */
 204         NUM_DT_COLUMNS
 205 };
 206
 207 static struct osl_column_description dir_table_cols[] = {
 208         [DT_NAME] = {
 209                 .storage_type = OSL_MAPPED_STORAGE,
 210                 .storage_flags = 0,
 211                 .name = "dir",
 212         },
 213         [DT_NUM] = {
 214                 .storage_type = OSL_MAPPED_STORAGE,
 215                 .storage_flags = OSL_RBTREE | OSL_FIXED_SIZE | OSL_UNIQUE,
 216                 .name = "num",
 217                 .compare_function = uint64_compare,
 218                 .data_size = sizeof(uint64_t)
 219         },
 220         [DT_BYTES] = {
 221                 .storage_type = OSL_MAPPED_STORAGE,
 222                 .storage_flags =  OSL_RBTREE | OSL_FIXED_SIZE,
 223                 .compare_function = size_compare,
 224                 .name = "num_bytes",
 225                 .data_size = sizeof(uint64_t)
 226         },
 227         [DT_FILES] = {
 228                 .storage_type = OSL_MAPPED_STORAGE,
 229                 .storage_flags =  OSL_RBTREE | OSL_FIXED_SIZE,
 230                 .compare_function = size_compare,
 231                 .name = "num_files",
 232                 .data_size = sizeof(uint64_t)
 233         }
 234 };
 235
 236 static struct osl_table_description dir_table_desc = {
 237         .name = "dir_table",
 238         .num_columns = NUM_DT_COLUMNS,
 239         .flags = 0,
 240         .column_descriptions = dir_table_cols,
 241 };
 242
 243 /** The columns of the id table. */
 244 enum user_table_columns {
 245         /** The numer of the directory. */
 246         UT_DIR_NUM,
 247         /** The number of bytes of all regular files in this dir owned by this id. */
 248         UT_BYTES,
 249         /** The number of files in this dir owned by this id. */
 250         UT_FILES,
 251         /** Number of columns in this table. */
 252         NUM_UT_COLUMNS
 253 };
 254
 255 static struct osl_column_description user_table_cols[] = {
 256         [UT_DIR_NUM] = {
 257                 .storage_type = OSL_MAPPED_STORAGE,
 258                 .storage_flags = OSL_RBTREE | OSL_FIXED_SIZE | OSL_UNIQUE,
 259                 .name = "dir_num",
 260                 .compare_function = uint64_compare,
 261                 .data_size = sizeof(uint64_t)
 262         },
 263         [UT_BYTES] = {
 264                 .storage_type = OSL_MAPPED_STORAGE,
 265                 .storage_flags = OSL_RBTREE | OSL_FIXED_SIZE,
 266                 .compare_function = size_compare,
 267                 .name = "num_bytes",
 268                 .data_size = sizeof(uint64_t)
 269         },
 270         [UT_FILES] = {
 271                 .storage_type = OSL_MAPPED_STORAGE,
 272                 .storage_flags = OSL_RBTREE | OSL_FIXED_SIZE,
 273                 .compare_function = size_compare,
 274                 .name = "num_files",
 275                 .data_size = sizeof(uint64_t)
 276         },
 277 };
 278
 279 static struct osl_table *dir_table;
 280
 281 static int add_directory(char *dirname, uint64_t dir_num, uint64_t *dir_size,
 282                 uint64_t *dir_files)
 283 {
 284         struct osl_object dir_objects[NUM_DT_COLUMNS];
 285
 286         INFO_LOG("adding #%llu: %s\n", (long long unsigned)dir_num, dirname);
 287         dir_objects[DT_NAME].data = dirname;
 288         dir_objects[DT_NAME].size = strlen(dirname) + 1;
 289         dir_objects[DT_NUM].data = &dir_num;
 290         dir_objects[DT_NUM].size = sizeof(dir_num);
 291         dir_objects[DT_BYTES].data = dir_size;
 292         dir_objects[DT_BYTES].size = sizeof(*dir_size);
 293         dir_objects[DT_FILES].data = dir_files;
 294         dir_objects[DT_FILES].size = sizeof(*dir_files);
 295
 296         return osl_add_row(dir_table, dir_objects);
 297 }
 298
 299 static uint32_t num_uids;
 300
 301 static int open_user_table(struct user_info *ui, int create)
 302 {
 303         int ret;
 304
 305         ui->desc = para_malloc(sizeof(*ui->desc));
 306         ui->desc->num_columns = NUM_UT_COLUMNS;
 307         ui->desc->flags = 0;
 308         ui->desc->column_descriptions = user_table_cols;
 309         ui->desc->dir = para_strdup(conf.database_dir_arg);
 310         ui->desc->name = make_message("%u", (unsigned)ui->uid);
 311         INFO_LOG(".............................uid #%u: %u\n",
 312                 (unsigned)num_uids, (unsigned)ui->uid);
 313         if (create) {
 314                 ret = osl_create_table(ui->desc);
 315                 if (ret < 0)
 316                         goto err;
 317                 num_uids++;
 318         }
 319         ret = osl_open_table(ui->desc, &ui->table);
 320         if (ret < 0)
 321                 goto err;
 322         return 1;
 323 err:
 324         free((char *)ui->desc->name);
 325         free((char *)ui->desc->dir);
 326         free(ui->desc);
 327         ui->desc->name = NULL;
 328         ui->desc->dir = NULL;
 329         ui->desc = NULL;
 330         ui->table = NULL;
 331         ui->flags = 0;
 332         return ret;
 333 }
 334
 335 #define uid_hash_bits 8
 336 static uint32_t uid_hash_table_size = 1 << uid_hash_bits;
 337 #define PRIME1 0x811c9dc5
 338 #define PRIME2 0x01000193
 339
 340 static void create_hash_table(void)
 341 {
 342         uid_hash_table = para_calloc(uid_hash_table_size
 343                 * sizeof(struct user_info));
 344 }
 345
 346 static void free_hash_table(void)
 347 {
 348         free(uid_hash_table);
 349         uid_hash_table = NULL;
 350 }
 351
 352 static int create_tables(void)
 353 {
 354         int ret;
 355
 356         dir_table_desc.dir = para_strdup(conf.database_dir_arg);
 357         ret = osl_create_table(&dir_table_desc);
 358         if (ret < 0)
 359                 return ret;
 360         create_hash_table();
 361         return 1;
 362 }
 363
 364 /*
 365  * We use a hash table of size s=2^uid_hash_bits to map the uids into the
 366  * interval [0..s]. Hash collisions are treated by open addressing, i.e.
 367  * unused slots in the table are used to store different uids that hash to the
 368  * same slot.
 369  *
 370  * If a hash collision occurs, different slots are successively probed in order
 371  * to find an unused slot for the new uid. Probing is implemented via a second
 372  * hash function that maps the uid to h=(uid * PRIME2) | 1, which is always an
 373  * odd number.
 374  *
 375  * An odd number is sufficient to make sure each entry of the hash table gets
 376  * probed for probe_num between 0 and s-1 because s is a power of two, hence
 377  * the second hash value has never a common divisor with the hash table size.
 378  * IOW: h is invertible in the ring [0..s].
 379  */
 380 static uint32_t double_hash(uint32_t uid, uint32_t probe_num)
 381 {
 382         return (uid * PRIME1 + ((uid * PRIME2) | 1) * probe_num)
 383                 % uid_hash_table_size;
 384 }
 385
 386 #define FOR_EACH_USER(ui) for (ui = uid_hash_table; ui && ui < uid_hash_table \
 387                 + uid_hash_table_size; ui++)
 388
 389 enum search_uid_flags {
 390         OPEN_USER_TABLE = 1,
 391         CREATE_USER_TABLE = 2,
 392 };
 393
 394 static int uid_is_admissible(uint32_t uid)
 395 {
 396         int i;
 397
 398         for (i = 0; i < conf.uid_given; i++) {
 399                 struct uid_range *ur = admissible_uids + i;
 400
 401                 if (ur->low <= uid && ur->high >= uid)
 402                         break;
 403         }
 404         i = !conf.uid_given || i < conf.uid_given;
 405         DEBUG_LOG("uid %u is %sadmissible\n", (unsigned)uid,
 406                 i? "" : "not ");
 407         return i;
 408 }
 409
 410 static int search_uid(uint32_t uid, enum search_uid_flags flags,
 411                 struct user_info **ui_ptr)
 412 {
 413         uint32_t p;
 414
 415         for (p = 0; p < uid_hash_table_size; p++) {
 416                 struct user_info *ui = uid_hash_table + double_hash(uid, p);
 417
 418                 if (!ui_used(ui)) {
 419                         int ret;
 420                         if (!flags)
 421                                 return -E_BAD_UID;
 422                         ui->uid = uid;
 423                         ui->flags |= UI_FL_SLOT_USED;
 424                         if (!uid_is_admissible(uid))
 425                                 return 0;
 426                         ui->flags |= UI_FL_ADMISSIBLE;
 427                         ret = open_user_table(ui, flags & CREATE_USER_TABLE);
 428                         if (ret < 0)
 429                                 return ret;
 430
 431                         if (ui_ptr)
 432                                 *ui_ptr = ui;
 433                         return 1;
 434                 }
 435                 if (ui->uid != uid)
 436                         continue;
 437                 if (ui_ptr)
 438                         *ui_ptr = ui;
 439                 return 0;
 440         }
 441         return flags? -E_HASH_TABLE_OVERFLOW : -E_BAD_UID;
 442 }
 443
 444 static int update_user_row(struct osl_table *t, uint64_t dir_num,
 445                 uint64_t *add)
 446 {
 447         struct osl_row *row;
 448         struct osl_object obj = {.data = &dir_num, .size = sizeof(dir_num)};
 449
 450         int ret = osl_get_row(t, UT_DIR_NUM, &obj, &row);
 451
 452         if (ret < 0 && ret != -E_RB_KEY_NOT_FOUND)
 453                 return ret;
 454         if (ret < 0) { /* this is the first file we add */
 455                 struct osl_object objects[NUM_UT_COLUMNS];
 456                 uint64_t num_files = 1;
 457
 458                 objects[UT_DIR_NUM].data = &dir_num;
 459                 objects[UT_DIR_NUM].size = sizeof(dir_num);
 460                 objects[UT_BYTES].data = add;
 461                 objects[UT_BYTES].size = sizeof(*add);
 462                 objects[UT_FILES].data = &num_files;
 463                 objects[UT_FILES].size = sizeof(num_files);
 464                 INFO_LOG("######################### ret: %d\n", ret);
 465                 ret = osl_add_row(t, objects);
 466                 INFO_LOG("######################### ret: %d\n", ret);
 467                 return ret;
 468         } else { /* add size and increment file count */
 469                 uint64_t num;
 470                 struct osl_object obj1, obj2 = {.data = &num, .size = sizeof(num)};
 471
 472                 ret = osl_get_object(t, row, UT_BYTES, &obj1);
 473                 if (ret < 0)
 474                         return ret;
 475                 num = *(uint64_t *)obj1.data + *add;
 476                 ret = osl_update_object(t, row, UT_BYTES, &obj2);
 477                 if (ret < 0)
 478                         return ret;
 479                 ret = osl_get_object(t, row, UT_FILES, &obj1);
 480                 if (ret < 0)
 481                         return ret;
 482                 num = *(uint64_t *)obj1.data + 1;
 483                 return osl_update_object(t, row, UT_FILES, &obj2);
 484         }
 485 }
 486
 487 static uint64_t num_dirs;
 488 static uint64_t num_files;
 489 static uint64_t num_bytes;
 490
 491 int scan_dir(char *dirname)
 492 {
 493         DIR *dir;
 494         struct dirent *entry;
 495         int ret, cwd_fd, ret2;
 496         uint64_t dir_size = 0, dir_files = 0;
 497         uint64_t this_dir_num = num_dirs++;
 498
 499         DEBUG_LOG("----------------- %llu: %s\n", (long long unsigned)num_dirs, dirname);
 500         ret = para_opendir(dirname, &dir, &cwd_fd);
 501         if (ret < 0) {
 502                 if (ret != -ERRNO_TO_ERROR(EACCES))
 503                         return ret;
 504                 WARNING_LOG("permission denied for %s\n", dirname);
 505                 return 1;
 506         }
 507         while ((entry = readdir(dir))) {
 508                 mode_t m;
 509                 char *tmp;
 510                 struct stat s;
 511                 uint32_t uid;
 512                 uint64_t size;
 513                 struct user_info *ui;
 514
 515                 if (!strcmp(entry->d_name, "."))
 516                         continue;
 517                 if (!strcmp(entry->d_name, ".."))
 518                         continue;
 519                 if (lstat(entry->d_name, &s) == -1) {
 520                         WARNING_LOG("lstat error for %s/%s\n", dirname,
 521                                 entry->d_name);
 522                         continue;
 523                 }
 524                 m = s.st_mode;
 525                 if (!S_ISREG(m) && !S_ISDIR(m))
 526                         continue;
 527                 if (S_ISDIR(m)) {
 528                         tmp = make_message("%s/%s", dirname, entry->d_name);
 529                         ret = scan_dir(tmp);
 530                         free(tmp);
 531                         if (ret < 0)
 532                                 goto out;
 533                         continue;
 534                 }
 535                 /* regular file */
 536                 size = s.st_size;
 537                 dir_size += size;
 538                 num_bytes += size;
 539                 dir_files++;
 540                 num_files++;
 541                 uid = s.st_uid;
 542                 ret = search_uid(uid, CREATE_USER_TABLE | OPEN_USER_TABLE, &ui);
 543                 if (ret < 0)
 544                         goto out;
 545                 ui->bytes += size;
 546                 ui->files++;
 547                 ret = update_user_row(ui->table, this_dir_num, &size);
 548                 if (ret < 0)
 549                         goto out;
 550         }
 551         ret = add_directory(dirname, this_dir_num, &dir_size, &dir_files);
 552 out:
 553         closedir(dir);
 554         ret2 = para_fchdir(cwd_fd);
 555         if (ret2 < 0 && ret >= 0)
 556                 ret = ret2;
 557         close(cwd_fd);
 558         return ret;
 559 }
 560
 561 static int get_dir_name(struct osl_row *row, char **name)
 562 {
 563         struct osl_object obj;
 564         int ret = osl_get_object(dir_table, row, DT_NAME, &obj);
 565
 566         if (ret < 0)
 567                 return ret;
 568         *name = obj.data;
 569         return 1;
 570 }
 571
 572 const uint64_t size_unit_divisors[] = {
 573         [size_unit_arg_b] = 1ULL,
 574         [size_unit_arg_k] = 1024ULL,
 575         [size_unit_arg_m] = 1024ULL * 1024ULL,
 576         [size_unit_arg_g] = 1024ULL * 1024ULL * 1024ULL,
 577         [size_unit_arg_t] = 1024ULL * 1024ULL * 1024ULL * 1024ULL,
 578 };
 579
 580 const uint64_t count_unit_divisors[] = {
 581
 582         [count_unit_arg_n] = 1ULL,
 583         [count_unit_arg_k] = 1000ULL,
 584         [count_unit_arg_m] = 1000ULL * 1000ULL,
 585         [count_unit_arg_g] = 1000ULL * 1000ULL * 1000ULL,
 586         [count_unit_arg_t] = 1000ULL * 1000ULL * 1000ULL * 1000ULL,
 587 };
 588
 589 const char size_unit_abbrevs[] = " BKMGT";
 590 const char count_unit_abbrevs[] = "  KMGT";
 591
 592 static void format_size_value(enum enum_size_unit unit, uint64_t value, char *result)
 593 {
 594         if (unit == size_unit_arg_h) /* human readable */
 595                 for (unit = size_unit_arg_b; unit < size_unit_arg_t && value > size_unit_divisors[unit + 1]; unit++)
 596                                 ; /* nothing */
 597         sprintf(result, "%llu%c", (long long unsigned)value / size_unit_divisors[unit], size_unit_abbrevs[unit]);
 598 }
 599
 600 static void format_count_value(enum enum_count_unit unit, uint64_t value, char *result)
 601 {
 602         if (unit == count_unit_arg_h) /* human readable */
 603                 for (unit = count_unit_arg_n; unit < count_unit_arg_t && value > count_unit_divisors[unit + 1]; unit++)
 604                                 ; /* nothing */
 605         sprintf(result, "%llu%c", (long long unsigned)value / count_unit_divisors[unit], count_unit_abbrevs[unit]);
 606 }
 607
 608 enum global_stats_flags {
 609         GSF_PRINT_DIRNAME = 1,
 610         GSF_PRINT_BYTES = 2,
 611         GSF_PRINT_FILES = 4,
 612         GSF_COMPUTE_SUMMARY = 8,
 613 };
 614
 615 struct global_stats_info {
 616         uint32_t count;
 617         enum global_stats_flags flags;
 618 };
 619
 620 static int global_stats_loop_function(struct osl_row *row, void *data)
 621 {
 622         struct global_stats_info *gsi = data;
 623         struct osl_object obj;
 624         char *dirname, formated_value[25];
 625         int ret, summary = gsi->flags & GSF_COMPUTE_SUMMARY;
 626
 627         if (!gsi->count && !summary)
 628                 return -E_LOOP_COMPLETE;
 629         if (gsi->count && (gsi->flags & GSF_PRINT_DIRNAME)) {
 630                 ret = get_dir_name(row, &dirname);
 631                 if (ret < 0)
 632                         return ret;
 633                 printf("%s%s", dirname,
 634                         (gsi->flags & (GSF_PRINT_FILES | GSF_PRINT_BYTES))?
 635                                 "\t" : "\n"
 636                 );
 637         }
 638         if (summary || (gsi->count && (gsi->flags & GSF_PRINT_FILES))) {
 639                 uint64_t files;
 640                 ret = osl_get_object(dir_table, row, DT_FILES, &obj);
 641                 if (ret < 0)
 642                         return ret;
 643                 files = *(uint64_t *)obj.data;
 644                 if (gsi->count && (gsi->flags & GSF_PRINT_FILES)) {
 645                         format_size_value(conf.size_unit_arg, files,
 646                                 formated_value);
 647                         printf("%s%s", formated_value,
 648                                 (gsi->flags & GSF_PRINT_BYTES)? "\t" : "\n");
 649                 }
 650                 if (summary)
 651                         num_files += files;
 652         }
 653         if (summary || (gsi->count && (gsi->flags & GSF_PRINT_BYTES))) {
 654                 uint64_t bytes;
 655                 ret = osl_get_object(dir_table, row, DT_BYTES, &obj);
 656                 if (ret < 0)
 657                         return ret;
 658                 bytes = *(uint64_t *)obj.data;
 659                 if (gsi->count && (gsi->flags & GSF_PRINT_BYTES)) {
 660                         format_size_value(conf.size_unit_arg, bytes,
 661                                 formated_value);
 662                         printf("%s\n", formated_value);
 663                 }
 664                 if (summary) {
 665                         num_bytes += bytes;
 666                         num_dirs++;
 667                 }
 668         }
 669         if (gsi->count > 0)
 670                 gsi->count--;
 671         return 1;
 672 }
 673
 674 static void print_id_stats(void)
 675 {
 676         struct user_info *ui;
 677
 678         printf("--------------------- user summary (uid/dirs/files/bytes):\n");
 679         FOR_EACH_USER(ui) {
 680                 char formated_dir_count[25], formated_file_count[25],
 681                         formated_bytes[25];
 682                 if (!ui_used(ui))
 683                         continue;
 684                 format_count_value(conf.count_unit_arg, ui->dirs,
 685                         formated_dir_count);
 686                 format_count_value(conf.count_unit_arg, ui->files,
 687                         formated_file_count);
 688                 format_size_value(conf.size_unit_arg, ui->bytes,
 689                         formated_bytes);
 690                 printf("%u\t%s\t%s\t%s\n", (unsigned)ui->uid,
 691                         formated_dir_count,
 692                         formated_file_count,
 693                         formated_bytes
 694                 );
 695         }
 696 }
 697
 698 enum user_stats_flags {
 699         USF_PRINT_DIRNAME = 1,
 700         USF_PRINT_BYTES = 2,
 701         USF_PRINT_FILES = 4,
 702         USF_COMPUTE_SUMMARY = 8,
 703 };
 704
 705 struct user_stats_info {
 706         uint32_t count;
 707         enum user_stats_flags flags;
 708         struct user_info *ui;
 709 };
 710
 711 static int user_stats_loop_function(struct osl_row *row, void *data)
 712 {
 713         struct user_stats_info *usi = data;
 714         struct osl_row *dir_row;
 715         struct osl_object obj;
 716         int ret, summary = usi->flags & GSF_COMPUTE_SUMMARY;
 717         char formated_value[25];
 718
 719         if (!usi->count && !summary)
 720                 return -E_LOOP_COMPLETE;
 721         if (usi->count && (usi->flags & USF_PRINT_DIRNAME)) {
 722                 char *dirname;
 723                 ret = osl_get_object(usi->ui->table, row, UT_DIR_NUM, &obj);
 724                 if (ret < 0)
 725                         return ret;
 726                 ret = osl_get_row(dir_table, DT_NUM, &obj, &dir_row);
 727                 if (ret < 0)
 728                         return ret;
 729                 ret = osl_get_object(dir_table, dir_row, DT_NAME, &obj);
 730                 if (ret < 0)
 731                         return ret;
 732                 dirname = obj.data;
 733                 printf("%s%s",
 734                         dirname,
 735                         (usi->flags & (USF_PRINT_FILES | USF_PRINT_BYTES))?
 736                                 "\t" : "\n"
 737                 );
 738         }
 739         if (summary || (usi->count && (usi->flags & USF_PRINT_FILES))) {
 740                 uint64_t files;
 741                 ret = osl_get_object(usi->ui->table, row, UT_FILES, &obj);
 742                 if (ret < 0)
 743                         return ret;
 744                 files = *(uint64_t *)obj.data;
 745                 if (usi->count && (usi->flags & USF_PRINT_FILES)) {
 746                         format_size_value(conf.size_unit_arg, files,
 747                                 formated_value);
 748                         printf("%s%s", formated_value,
 749                                 (usi->flags & USF_PRINT_BYTES)? "\t" : "\n"
 750                         );
 751                 }
 752                 if (summary)
 753                         usi->ui->files += files;
 754         }
 755         if (summary || (usi->count && (usi->flags & USF_PRINT_BYTES))) {
 756                 uint64_t bytes;
 757                 ret = osl_get_object(usi->ui->table, row, UT_BYTES, &obj);
 758                 if (ret < 0)
 759                         return ret;
 760                 bytes = *(uint64_t *)obj.data;
 761                 if (usi->count && (usi->flags & USF_PRINT_BYTES)) {
 762                         format_size_value(conf.size_unit_arg, bytes,
 763                                 formated_value);
 764                         printf("%s\n", formated_value);
 765                 }
 766                 if (summary) {
 767                         usi->ui->bytes += bytes;
 768                         usi->ui->dirs++;
 769                 }
 770
 771         }
 772         if (usi->count > 0)
 773                 usi->count--;
 774         return 1;
 775 }
 776
 777 static void print_user_stats(void)
 778 {
 779         struct user_info *ui;
 780
 781         FOR_EACH_USER(ui) {
 782                 struct user_stats_info usi = {
 783                         .count = conf.limit_arg,
 784                         .ui = ui
 785                 };
 786                 if (!ui_used(ui) || !ui_admissible(ui))
 787                         continue;
 788                 usi.flags = USF_PRINT_DIRNAME | USF_PRINT_BYTES | USF_COMPUTE_SUMMARY;
 789                 printf("************************************************ uid %u\n",
 790                         (unsigned) ui->uid);
 791                 printf("----------------- Largest dirs -------------------\n");
 792                 osl_rbtree_loop_reverse(ui->table, UT_BYTES, &usi,
 793                         user_stats_loop_function);
 794                 printf("---------- dirs containing most files ------------\n");
 795                 usi.count = conf.limit_arg,
 796                 usi.flags = USF_PRINT_DIRNAME | USF_PRINT_FILES;
 797                 osl_rbtree_loop_reverse(ui->table, UT_FILES, &usi,
 798                         user_stats_loop_function);
 799         }
 800 }
 801
 802 static int print_statistics(void)
 803 {
 804         int ret;
 805         struct global_stats_info gsi = {
 806                 .count = conf.limit_arg,
 807                 .flags = GSF_PRINT_DIRNAME | GSF_PRINT_BYTES | GSF_COMPUTE_SUMMARY
 808         };
 809
 810         printf("----------------- Largest dirs -------------------\n");
 811         ret = osl_rbtree_loop_reverse(dir_table, DT_BYTES, &gsi,
 812                 global_stats_loop_function);
 813         if (ret < 0 && ret != -E_LOOP_COMPLETE)
 814                 return ret;
 815         gsi.count = conf.limit_arg;
 816
 817         gsi.flags = GSF_PRINT_DIRNAME | GSF_PRINT_FILES;
 818         printf("---------- dirs containing most files ------------\n");
 819         ret = osl_rbtree_loop_reverse(dir_table, DT_FILES, &gsi,
 820                 global_stats_loop_function);
 821         if (ret < 0 && ret != -E_LOOP_COMPLETE)
 822                 return ret;
 823
 824         printf("------------------ Global summary (dirs/files/bytes)\n"
 825                 "%llu\t%llu\t%llu\n",
 826                 (long long unsigned)num_dirs, (long long unsigned)num_files,
 827                 (long long unsigned)num_bytes);
 828         print_user_stats();
 829         print_id_stats();
 830         return 1;
 831 }
 832
 833 static char *get_uid_list_name(void)
 834 {
 835         return make_message("%s/uid_list", conf.database_dir_arg);
 836 }
 837
 838 static int write_uid_list(void)
 839 {
 840         char *buf, *filename;
 841         uint32_t count = 0;
 842         struct user_info *ui;
 843         size_t size = num_uids * sizeof(uint32_t);
 844         int ret;
 845
 846         if (!num_uids)
 847                 return 0;
 848         buf = para_malloc(size);
 849         FOR_EACH_USER(ui) {
 850                 if (!ui_used(ui) || !ui_admissible(ui))
 851                         continue;
 852                 DEBUG_LOG("saving uid %u\n", (unsigned) ui->uid);
 853                 write_u32(buf + count++ * sizeof(uint32_t), ui->uid);
 854         }
 855         filename = get_uid_list_name();
 856         ret = para_write_file(filename, buf, size);
 857         free(filename);
 858         free(buf);
 859         return ret;
 860 }
 861
 862 static int open_dir_table(void)
 863 {
 864         if (!dir_table_desc.dir) /* we did not create the table */
 865                 dir_table_desc.dir = para_strdup(conf.database_dir_arg);
 866         return osl_open_table(&dir_table_desc, &dir_table);
 867 }
 868
 869 static void close_dir_table(void)
 870 {
 871         int ret;
 872
 873         if (!dir_table)
 874                 return;
 875         ret = osl_close_table(dir_table, OSL_MARK_CLEAN);
 876         if (ret < 0)
 877                 ERROR_LOG("failed to close dir table: %s\n", error_txt(-ret));
 878         free((char *)dir_table_desc.dir);
 879         dir_table = NULL;
 880 }
 881
 882 static void close_user_table(struct user_info *ui)
 883 {
 884         int ret;
 885
 886         if (!ui || !ui_used(ui) || !ui_admissible(ui))
 887                 return;
 888         ret = osl_close_table(ui->table, OSL_MARK_CLEAN);
 889         if (ret < 0)
 890                 ERROR_LOG("failed to close user table %u: %s\n",
 891                         (unsigned) ui->uid, error_txt(-ret));
 892         free((char *)ui->desc->name);
 893         ui->desc->name = NULL;
 894         free((char *)ui->desc->dir);
 895         ui->desc->dir = NULL;
 896         free(ui->desc);
 897         ui->desc = NULL;
 898         ui->table = NULL;
 899         ui->flags = 0;
 900 }
 901
 902 static void close_user_tables(void)
 903 {
 904         struct user_info *ui;
 905
 906         FOR_EACH_USER(ui)
 907                 close_user_table(ui);
 908 }
 909
 910 static void close_all_tables(void)
 911 {
 912         close_dir_table();
 913         close_user_tables();
 914         free_hash_table();
 915 }
 916
 917 static int com_create()
 918 {
 919         int ret = create_tables();
 920         if (ret < 0)
 921                 return ret;
 922         ret = open_dir_table();
 923         if (ret < 0)
 924                 return ret;
 925         ret = scan_dir(conf.base_dir_arg);
 926         if (ret < 0)
 927                 goto out;
 928         ret = write_uid_list();
 929 out:
 930         close_all_tables();
 931         return ret;
 932 }
 933
 934 static int read_uid_file(void)
 935 {
 936         size_t size;
 937         uint32_t n;
 938         char *filename = get_uid_list_name(), *map;
 939         int ret = mmap_full_file(filename, O_RDONLY, (void **)&map, &size, NULL);
 940
 941         if (ret < 0) {
 942                 INFO_LOG("failed to map %s\n", filename);
 943                 free(filename);
 944                 return ret;
 945         }
 946         num_uids = size / 4;
 947         INFO_LOG("found %u uids in %s\n", (unsigned)num_uids, filename);
 948         free(filename);
 949         /* hash table size should be a power of two and larger than the number of uids */
 950         uid_hash_table_size = 4;
 951         while (uid_hash_table_size < num_uids)
 952                 uid_hash_table_size *= 2;
 953         create_hash_table();
 954         for (n = 0; n < num_uids; n++) {
 955                 uint32_t uid = read_u32(map + n * sizeof(uid));
 956                 ret = search_uid(uid, OPEN_USER_TABLE, NULL);
 957                 if (ret < 0)
 958                         goto out;
 959         }
 960 out:
 961         para_munmap(map, size);
 962         return ret;
 963 }
 964
 965 static int com_select(void)
 966 {
 967         int ret;
 968
 969         ret = open_dir_table();
 970         if (ret < 0)
 971                 return ret;
 972         ret = read_uid_file();
 973         if (ret < 0)
 974                 return ret;
 975         ret = print_statistics();
 976         close_all_tables();
 977         return ret;
 978 }
 979
 980 static int check_args(void)
 981 {
 982         int i, ret;
 983
 984         if (!conf.uid_given)
 985                 return 0;
 986
 987         admissible_uids = para_malloc(conf.uid_given * sizeof(*admissible_uids));
 988
 989         for (i = 0; i < conf.uid_given; i++) {
 990                 ret = parse_uid_range(conf.uid_arg[i], admissible_uids + i);
 991                 if (ret < 0)
 992                         goto err;
 993         }
 994         return 1;
 995 err:
 996         free(admissible_uids);
 997         admissible_uids = NULL;
 998         return ret;
 999 }
1000
1001 int main(int argc, char **argv)
1002 {
1003         int ret;
1004         struct cmdline_parser_params params = {
1005                 .override = 0,
1006                 .initialize = 1,
1007                 .check_required = 0,
1008                 .check_ambiguity = 0,
1009                 .print_errors = 1
1010         };
1011
1012         cmdline_parser_ext(argc, argv, &conf, &params); /* aborts on errors */
1013         ret = check_args();
1014         if (ret < 0)
1015                 goto out;
1016         ret = -E_SYNTAX;
1017         if (conf.select_given)
1018                 ret = com_select();
1019         else
1020                 ret = com_create();
1021         if (ret < 0)
1022                 goto out;
1023 out:
1024         free(admissible_uids);
1025         if (ret < 0) {
1026                 ERROR_LOG("%s\n", error_txt(-ret));
1027                 return -EXIT_FAILURE;
1028         }
1029         return EXIT_SUCCESS;
1030 }