]> git.tuebingen.mpg.de Git - dss.git/blob - dss.c
Get rid of E_INVALID_COMMAND.
[dss.git] / dss.c
1 #include <string.h>
2 #include <stdlib.h>
3 #include <stdarg.h>
4 #include <assert.h>
5 #include <errno.h>
6 #include <sys/types.h>
7 #include <signal.h>
8 #include <ctype.h>
9 #include <sys/stat.h>
10 #include <unistd.h>
11 #include <inttypes.h>
12 #include <time.h>
13 #include <sys/wait.h>
14 #include <fnmatch.h>
15 #include <limits.h>
16
17
18 #include "gcc-compat.h"
19 #include "cmdline.h"
20 #include "log.h"
21 #include "string.h"
22 #include "error.h"
23 #include "fd.h"
24 #include "exec.h"
25
26
27 struct gengetopt_args_info conf;
28 char *dss_error_txt = NULL;
29
30 DEFINE_DSS_ERRLIST;
31
32
33 /* a litte cpp magic helps to DRY */
34 #define COMMANDS \
35         COMMAND(ls) \
36         COMMAND(create) \
37         COMMAND(prune) \
38         COMMAND(run)
39 #define COMMAND(x) int com_ ##x(int, char * const * const);
40 COMMANDS
41 #undef COMMAND
42 #define COMMAND(x) if (conf.x ##_given) return com_ ##x(argc, argv);
43 int call_command_handler(int argc, char * const * const argv)
44 {
45         COMMANDS
46         DSS_EMERG_LOG("BUG: did not find command handler\n");
47         exit(EXIT_FAILURE);
48 }
49 #undef COMMAND
50 #undef COMMANDS
51
52 /*
53  * complete, not being deleted: 1204565370-1204565371.Sun_Mar_02_2008_14_33-Sun_Mar_02_2008_14_43
54  * complete, being deleted: 1204565370-1204565371.being_deleted
55  * incomplete, not being deleted: 1204565370-incomplete
56  * incomplete, being deleted: 1204565370-incomplete.being_deleted
57  */
58 enum snapshot_status_flags {
59         SS_COMPLETE = 1,
60         SS_BEING_DELETED = 2,
61 };
62
63 struct snapshot {
64         char *name;
65         int64_t creation_time;
66         int64_t completion_time;
67         enum snapshot_status_flags flags;
68         unsigned interval;
69 };
70
71 int is_snapshot(const char *dirname, int64_t now, struct snapshot *s)
72 {
73         int i, ret;
74         char *dash, *dot, *tmp;
75         int64_t num;
76
77         assert(dirname);
78         dash = strchr(dirname, '-');
79         if (!dash || !dash[1] || dash == dirname)
80                 return 0;
81         for (i = 0; dirname[i] != '-'; i++)
82                 if (!isdigit(dirname[i]))
83                         return 0;
84         tmp = dss_strdup(dirname);
85         tmp[i] = '\0';
86         ret = dss_atoi64(tmp, &num);
87         free(tmp);
88         if (ret < 0) {
89                 free(dss_error_txt);
90                 return 0;
91         }
92         assert(num >= 0);
93         if (num > now)
94                 return 0;
95         s->creation_time = num;
96         //DSS_DEBUG_LOG("%s start time: %lli\n", dirname, (long long)s->creation_time);
97         s->interval = (long long) ((now - s->creation_time)
98                 / conf.unit_interval_arg / 24 / 3600);
99         if (!strcmp(dash + 1, "incomplete")) {
100                 s->completion_time = -1;
101                 s->flags = 0; /* neither complete, nor being deleted */
102                 goto success;
103         }
104         if (!strcmp(dash + 1, "incomplete.being_deleted")) {
105                 s->completion_time = -1;
106                 s->flags = SS_BEING_DELETED; /* mot cpmplete, being deleted */
107                 goto success;
108         }
109         tmp = dash + 1;
110         dot = strchr(tmp, '.');
111         if (!dot || !dot[1] || dot == tmp)
112                 return 0;
113         for (i = 0; tmp[i] != '.'; i++)
114                 if (!isdigit(tmp[i]))
115                         return 0;
116         tmp = dss_strdup(dash + 1);
117         tmp[i] = '\0';
118         ret = dss_atoi64(tmp, &num);
119         free(tmp);
120         if (ret < 0) {
121                 free(dss_error_txt);
122                 return 0;
123         }
124         if (num > now)
125                 return 0;
126         s->completion_time = num;
127         s->flags = SS_COMPLETE;
128         if (strcmp(dot + 1, "being_deleted"))
129                 s->flags |= SS_BEING_DELETED;
130 success:
131         s->name = dss_strdup(dirname);
132         return 1;
133 }
134
135 int64_t get_current_time(void)
136 {
137         time_t now;
138         time(&now);
139         DSS_DEBUG_LOG("now: %lli\n", (long long) now);
140         return (int64_t)now;
141 }
142
143 char *incomplete_name(int64_t start)
144 {
145         return make_message("%lli-incomplete", (long long)start);
146 }
147
148 char *being_deleted_name(struct snapshot *s)
149 {
150         if (s->flags & SS_COMPLETE)
151                 return make_message("%lli-%lli.being_deleted",
152                         (long long)s->creation_time,
153                         (long long)s->completion_time);
154         return make_message("%lli-incomplete.being_deleted",
155                 (long long)s->creation_time);
156 }
157
158 int complete_name(int64_t start, int64_t end, char **result)
159 {
160         struct tm start_tm, end_tm;
161         time_t *start_seconds = (time_t *) (uint64_t *)&start; /* STFU, gcc */
162         time_t *end_seconds = (time_t *) (uint64_t *)&end; /* STFU, gcc */
163         char start_str[200], end_str[200];
164
165         if (!localtime_r(start_seconds, &start_tm)) {
166                 make_err_msg("%lli", (long long)start);
167                 return -E_LOCALTIME;
168         }
169         if (!localtime_r(end_seconds, &end_tm)) {
170                 make_err_msg("%lli", (long long)end);
171                 return -E_LOCALTIME;
172         }
173         if (!strftime(start_str, sizeof(start_str), "%a_%b_%d_%Y_%H_%M_%S", &start_tm)) {
174                 make_err_msg("%lli", (long long)start);
175                 return -E_STRFTIME;
176         }
177         if (!strftime(end_str, sizeof(end_str), "%a_%b_%d_%Y_%H_%M_%S", &end_tm)) {
178                 make_err_msg("%lli", (long long)end);
179                 return -E_STRFTIME;
180         }
181         *result = make_message("%lli-%lli.%s-%s", (long long) start, (long long) end,
182                 start_str, end_str);
183         return 1;
184 }
185
186 struct snapshot_list {
187         int64_t now;
188         unsigned num_snapshots;
189         unsigned array_size;
190         struct snapshot **snapshots;
191         /**
192          * Array of size num_intervals + 1
193          *
194          * It contains the number of snapshots in each interval. interval_count[num_intervals]
195          * is the number of snapshots which belong to any interval greater than num_intervals.
196          */
197         unsigned *interval_count;
198 };
199
200 #define FOR_EACH_SNAPSHOT(s, i, sl) \
201         for ((i) = 0; (i) < (sl)->num_snapshots && ((s) = (sl)->snapshots[(i)]); (i)++)
202
203
204
205 #define NUM_COMPARE(x, y) ((int)((x) < (y)) - (int)((x) > (y)))
206
207 static int compare_snapshots(const void *a, const void *b)
208 {
209         struct snapshot *s1 = *(struct snapshot **)a;
210         struct snapshot *s2 = *(struct snapshot **)b;
211         return NUM_COMPARE(s2->creation_time, s1->creation_time);
212 }
213
214 /** Compute the minimum of \a a and \a b. */
215 #define DSS_MIN(a,b) ((a) < (b) ? (a) : (b))
216
217 int add_snapshot(const char *dirname, void *private)
218 {
219         struct snapshot_list *sl = private;
220         struct snapshot s;
221         int ret = is_snapshot(dirname, sl->now, &s);
222
223         if (!ret)
224                 return 1;
225         if (sl->num_snapshots >= sl->array_size) {
226                 sl->array_size = 2 * sl->array_size + 1;
227                 sl->snapshots = dss_realloc(sl->snapshots,
228                         sl->array_size * sizeof(struct snapshot *));
229         }
230         sl->snapshots[sl->num_snapshots] = dss_malloc(sizeof(struct snapshot));
231         *(sl->snapshots[sl->num_snapshots]) = s;
232         sl->interval_count[DSS_MIN(s.interval, conf.num_intervals_arg)]++;
233         sl->num_snapshots++;
234         return 1;
235 }
236
237 void get_snapshot_list(struct snapshot_list *sl)
238 {
239         sl->now = get_current_time();
240         sl->num_snapshots = 0;
241         sl->array_size = 0;
242         sl->snapshots = NULL;
243         sl->interval_count = dss_calloc((conf.num_intervals_arg + 1) * sizeof(unsigned));
244         for_each_subdir(add_snapshot, sl);
245         qsort(sl->snapshots, sl->num_snapshots, sizeof(struct snapshot *),
246                 compare_snapshots);
247 }
248
249 void free_snapshot_list(struct snapshot_list *sl)
250 {
251         int i;
252         struct snapshot *s;
253
254         FOR_EACH_SNAPSHOT(s, i, sl) {
255                 free(s->name);
256                 free(s);
257         }
258         free(sl->interval_count);
259         free(sl->snapshots);
260 }
261
262 /**
263  * Print a log message about the exit status of a child.
264  */
265 void log_termination_msg(pid_t pid, int status)
266 {
267         if (WIFEXITED(status))
268                 DSS_INFO_LOG("child %i exited. Exit status: %i\n", (int)pid,
269                         WEXITSTATUS(status));
270         else if (WIFSIGNALED(status))
271                 DSS_NOTICE_LOG("child %i was killed by signal %i\n", (int)pid,
272                         WTERMSIG(status));
273         else
274                 DSS_WARNING_LOG("child %i terminated abormally\n", (int)pid);
275 }
276
277 int wait_for_process(pid_t pid, int *status)
278 {
279         int ret;
280
281         DSS_DEBUG_LOG("Waiting for process %d to terminate\n", (int)pid);
282         for (;;) {
283                 ret = waitpid(pid, status, 0);
284                 if (ret >= 0 || errno != EINTR)
285                         break;
286         }
287         if (ret < 0) {
288                 ret = -ERRNO_TO_DSS_ERROR(errno);
289                 make_err_msg("failed to wait for process %d", (int)pid);
290         } else
291                 log_termination_msg(pid, *status);
292         return ret;
293 }
294
295 int remove_snapshot(struct snapshot *s, pid_t *pid)
296 {
297         int fds[3] = {0, 0, 0};
298         char *new_name = being_deleted_name(s);
299         int ret = dss_rename(s->name, new_name);
300         char *argv[] = {"rm", "-rf", new_name, NULL};
301
302         if (ret < 0)
303                 goto out;
304         DSS_NOTICE_LOG("removing %s (interval = %i)\n", s->name, s->interval);
305         ret = dss_exec(pid, argv[0], argv, fds);
306 out:
307         free(new_name);
308         return ret;
309 }
310
311 int remove_redundant_snapshot(struct snapshot_list *sl,
312                 int dry_run, pid_t *pid)
313 {
314         int ret, i, interval;
315         struct snapshot *s;
316         unsigned missing = 0;
317
318         DSS_INFO_LOG("looking for intervals containing too many snapshots\n");
319         for (interval = conf.num_intervals_arg - 1; interval >= 0; interval--) {
320                 unsigned keep = 1<<(conf.num_intervals_arg - interval - 1);
321                 unsigned num = sl->interval_count[interval];
322                 struct snapshot *victim = NULL, *prev = NULL;
323                 int64_t score = LONG_MAX;
324
325                 if (keep >= num)
326                         missing += keep - num;
327                 DSS_DEBUG_LOG("interval %i: keep: %u, have: %u, missing: %u\n",
328                         interval, keep, num, missing);
329                 if (keep + missing >= num)
330                         continue;
331                 /* redundant snapshot in this interval, pick snapshot with lowest score */
332                 FOR_EACH_SNAPSHOT(s, i, sl) {
333                         int64_t this_score;
334
335                         DSS_DEBUG_LOG("checking %s\n", s->name);
336                         if (s->interval > interval) {
337                                 prev = s;
338                                 continue;
339                         }
340                         if (s->interval < interval)
341                                 break;
342                         if (!victim) {
343                                 victim = s;
344                                 prev = s;
345                                 continue;
346                         }
347                         assert(prev);
348                         /* check if s is a better victim */
349                         this_score = s->creation_time - prev->creation_time;
350                         assert(this_score >= 0);
351                         DSS_DEBUG_LOG("%s: score %lli\n", s->name, (long long)score);
352                         if (this_score < score) {
353                                 score = this_score;
354                                 victim = s;
355                         }
356                         prev = s;
357                 }
358                 assert(victim);
359                 if (dry_run) {
360                         printf("%s would be removed (interval = %i)\n",
361                                 victim->name, victim->interval);
362                         continue;
363                 }
364                 ret = remove_snapshot(victim, pid);
365                 return ret < 0? ret : 1;
366         }
367         return 0;
368 }
369
370 int remove_old_snapshot(struct snapshot_list *sl, int dry_run, pid_t *pid)
371 {
372         int i, ret;
373         struct snapshot *s;
374
375         DSS_INFO_LOG("looking for snapshots belonging to intervals greater than %d\n",
376                 conf.num_intervals_arg);
377         FOR_EACH_SNAPSHOT(s, i, sl) {
378                 if (s->interval <= conf.num_intervals_arg)
379                         continue;
380                 if (dry_run) {
381                         printf("%s would be removed (interval = %i)\n",
382                                 s->name, s->interval);
383                         continue;
384                 }
385                 ret = remove_snapshot(s, pid);
386                 if (ret < 0)
387                         return ret;
388                 return 1;
389         }
390         return 0;
391 }
392
393 int wait_for_rm_process(pid_t pid)
394 {
395         int status, es, ret = wait_for_process(pid, &status);
396         if (ret < 0)
397                 return ret;
398         if (!WIFEXITED(status)) {
399                 ret = E_INVOLUNTARY_EXIT;
400                 make_err_msg("rm process %d died involuntary", (int)pid);
401                 return ret;
402         }
403         es = WEXITSTATUS(status);
404         if (es) {
405                 ret = -E_BAD_EXIT_CODE;
406                 make_err_msg("rm process %d returned %d", (int)pid, es);
407                 return ret;
408         }
409         return 1;
410 }
411
412 int com_run(int argc, char * const * argv)
413 {
414         return 42;
415 }
416
417 int com_prune(int argc, char * const * argv)
418 {
419         int ret, dry_run = 0;
420         struct snapshot_list sl;
421         pid_t pid;
422
423         if (argc > 2) {
424                 make_err_msg("too many arguments");
425                 return -E_SYNTAX;
426         }
427         if (argc == 2) {
428                 if (strcmp(argv[1], "-d")) {
429                         make_err_msg("%s", argv[1]);
430                         return -E_SYNTAX;
431                 }
432                 dry_run = 1;
433         }
434         for (;;) {
435                 get_snapshot_list(&sl);
436                 ret = remove_old_snapshot(&sl, dry_run, &pid);
437                 free_snapshot_list(&sl);
438                 if (ret < 0)
439                         return ret;
440                 if (!ret)
441                         break;
442                 ret = wait_for_rm_process(pid);
443                 if (ret < 0)
444                         goto out;
445         }
446         for (;;) {
447                 get_snapshot_list(&sl);
448                 ret = remove_redundant_snapshot(&sl, dry_run, &pid);
449                 free_snapshot_list(&sl);
450                 if (ret < 0)
451                         return ret;
452                 if (!ret)
453                         break;
454                 ret = wait_for_rm_process(pid);
455                 if (ret < 0)
456                         goto out;
457         }
458         return 1;
459 out:
460         return ret;
461 }
462
463 struct newest_snapshot_data {
464         char * newest_name;
465         int64_t newest_creation_time;
466         int64_t now;
467 };
468
469 int get_newest_complete(const char *dirname, void *private)
470 {
471         struct newest_snapshot_data *nsd = private;
472         struct snapshot s;
473         int ret = is_snapshot(dirname, nsd->now, &s);
474
475         if (ret <= 0)
476                 return 1;
477         if (s.creation_time < nsd->newest_creation_time)
478                 return 1;
479         nsd->newest_creation_time = s.creation_time;
480         free(nsd->newest_name);
481         nsd->newest_name = s.name;
482         return 1;
483 }
484
485 __malloc char *name_of_newest_complete_snapshot(void)
486 {
487         struct newest_snapshot_data nsd = {
488                 .now = get_current_time(),
489                 .newest_creation_time = -1
490         };
491         for_each_subdir(get_newest_complete, &nsd);
492         return nsd.newest_name;
493 }
494
495 void create_rsync_argv(char ***argv, int64_t *num)
496 {
497         char *logname, *newest = name_of_newest_complete_snapshot();
498         int i = 0, j;
499
500         *argv = dss_malloc((15 + conf.rsync_option_given) * sizeof(char *));
501         (*argv)[i++] = dss_strdup("rsync");
502         (*argv)[i++] = dss_strdup("-aq");
503         (*argv)[i++] = dss_strdup("--delete");
504         for (j = 0; j < conf.rsync_option_given; j++)
505                 (*argv)[i++] = dss_strdup(conf.rsync_option_arg[j]);
506         if (newest) {
507                 DSS_INFO_LOG("using %s as reference snapshot\n", newest);
508                 (*argv)[i++] = make_message("--link-dest=../%s", newest);
509                 free(newest);
510         } else
511                 DSS_INFO_LOG("no previous snapshot found");
512         if (conf.exclude_patterns_given) {
513                 (*argv)[i++] = dss_strdup("--exclude-from");
514                 (*argv)[i++] = dss_strdup(conf.exclude_patterns_arg);
515
516         }
517         logname = dss_logname();
518         if (conf.remote_user_given && !strcmp(conf.remote_user_arg, logname))
519                 (*argv)[i++] = dss_strdup(conf.source_dir_arg);
520         else
521                 (*argv)[i++] = make_message("%s@%s:%s/", conf.remote_user_given?
522                         conf.remote_user_arg : logname,
523                         conf.remote_host_arg, conf.source_dir_arg);
524         free(logname);
525         *num = get_current_time();
526         (*argv)[i++] = incomplete_name(*num);
527         (*argv)[i++] = NULL;
528         for (j = 0; j < i; j++)
529                 DSS_DEBUG_LOG("argv[%d] = %s\n", j, (*argv)[j]);
530 }
531
532 void free_rsync_argv(char **argv)
533 {
534         int i;
535         for (i = 0; argv[i]; i++)
536                 free(argv[i]);
537         free(argv);
538 }
539
540 int create_snapshot(char **argv, pid_t *pid)
541 {
542         int fds[3] = {0, 0, 0};
543
544         return dss_exec(pid, argv[0], argv, fds);
545 }
546
547 int rename_incomplete_snapshot(int64_t start)
548 {
549         char *old_name, *new_name;
550         int ret;
551
552         ret = complete_name(start, get_current_time(), &new_name);
553         if (ret < 0)
554                 return ret;
555         old_name = incomplete_name(start);
556         ret = dss_rename(old_name, new_name);
557         if (ret >= 0)
558                 DSS_NOTICE_LOG("%s -> %s\n", old_name, new_name);
559         free(old_name);
560         free(new_name);
561         return ret;
562 }
563
564 int com_create(int argc, __a_unused char * const * argv)
565 {
566         int ret, status, es;
567         char **rsync_argv;
568         int64_t snapshot_num;
569         pid_t pid;
570
571         if (argc != 1) {
572                 ret = -E_SYNTAX;
573                 make_err_msg("create: no args expected, %d given", argc - 1);
574                 return ret;
575         }
576         create_rsync_argv(&rsync_argv, &snapshot_num);
577         DSS_NOTICE_LOG("creating snapshot %lli\n", (long long)snapshot_num);
578         ret = create_snapshot(rsync_argv, &pid);
579         if (ret < 0)
580                 goto out;
581         ret = wait_for_process(pid, &status);
582         if (ret < 0)
583                 goto out;
584         if (!WIFEXITED(status)) {
585                 ret = E_INVOLUNTARY_EXIT;
586                 make_err_msg("rsync process %d died involuntary", (int)pid);
587                 goto out;
588         }
589         es = WEXITSTATUS(status);
590         if (es != 0 && es != 23 && es != 24) {
591                 ret = -E_BAD_EXIT_CODE;
592                 make_err_msg("rsync process %d returned %d", (int)pid, es);
593                 goto out;
594         }
595         ret = rename_incomplete_snapshot(snapshot_num);
596 out:
597         free_rsync_argv(rsync_argv);
598         return ret;
599 }
600
601 int com_ls(int argc, __a_unused char * const * argv)
602 {
603         int i, ret;
604         struct snapshot_list sl;
605         struct snapshot *s;
606         if (argc != 1) {
607                 ret = -E_SYNTAX;
608                 make_err_msg("ls: no args expected, %d given", argc - 1);
609                 return ret;
610         }
611         get_snapshot_list(&sl);
612         FOR_EACH_SNAPSHOT(s, i, &sl)
613                 printf("%u\t%s\n", s->interval, s->name);
614         free_snapshot_list(&sl);
615         return 1;
616 }
617
618 /* TODO: Unlink pid file */
619 __noreturn void clean_exit(int status)
620 {
621         //kill(0, SIGTERM);
622         free(dss_error_txt);
623         exit(status);
624 }
625
626 __printf_2_3 void dss_log(int ll, const char* fmt,...)
627 {
628         va_list argp;
629         if (ll < conf.loglevel_arg)
630                 return;
631         va_start(argp, fmt);
632         vfprintf(stderr, fmt, argp);
633         va_end(argp);
634 }
635
636 int read_config_file(void)
637 {
638         int ret;
639         char *config_file;
640         struct stat statbuf;
641
642         if (conf.config_file_given)
643                 config_file = dss_strdup(conf.config_file_arg);
644         else {
645                 char *home = get_homedir();
646                 config_file = make_message("%s/.dssrc", home);
647                 free(home);
648         }
649         ret = stat(config_file, &statbuf);
650         if (ret && conf.config_file_given) {
651                 ret = -ERRNO_TO_DSS_ERROR(errno);
652                 make_err_msg("failed to stat config file %s", config_file);
653                 goto out;
654         }
655         if (!ret) {
656                 struct cmdline_parser_params params = {
657                         .override = 0,
658                         .initialize = 0,
659                         .check_required = 0,
660                         .check_ambiguity = 0
661                 };
662                 cmdline_parser_config_file(config_file, &conf, &params);
663         }
664         if (!conf.source_dir_given || !conf.dest_dir_given) {
665                 ret = -E_SYNTAX;
666                 make_err_msg("you need to specify both source_dir and dest_dir");
667                 goto out;
668         }
669         ret = 1;
670 out:
671         free(config_file);
672         return ret;
673 }
674
675 int check_config(void)
676 {
677         if (conf.unit_interval_arg <= 0) {
678                 make_err_msg("bad unit interval: %i", conf.unit_interval_arg);
679                 return -E_INVALID_NUMBER;
680         }
681         DSS_DEBUG_LOG("unit interval: %i day(s)\n", conf.unit_interval_arg);
682         if (conf.num_intervals_arg <= 0) {
683                 make_err_msg("bad number of intervals  %i", conf.num_intervals_arg);
684                 return -E_INVALID_NUMBER;
685         }
686         DSS_DEBUG_LOG("number of intervals: %i\n", conf.num_intervals_arg);
687         return 1;
688 }
689
690 int main(int argc, char **argv)
691 {
692         int ret;
693
694         cmdline_parser(argc, argv, &conf); /* aborts on errors */
695         if (conf.inputs_num) {
696                 ret = -E_SYNTAX;
697                 make_err_msg("additional non-options given");
698                 goto out;
699         }
700         ret = read_config_file();
701         if (ret < 0)
702                 goto out;
703         ret = check_config();
704         if (ret < 0)
705                 goto out;
706         ret = dss_chdir(conf.dest_dir_arg);
707         if (ret < 0)
708                 goto out;
709         ret = call_command_handler(conf.inputs_num, conf.inputs);
710 out:
711         if (ret < 0)
712                 log_err_msg(EMERG, -ret);
713         clean_exit(ret >= 0? EXIT_SUCCESS : EXIT_FAILURE);
714 }