]> git.tuebingen.mpg.de Git - dss.git/blob - dss.c
Implement logfile and daemon mode.
[dss.git] / dss.c
1 #include <string.h>
2 #include <stdlib.h>
3 #include <stdarg.h>
4 #include <assert.h>
5 #include <errno.h>
6 #include <sys/types.h>
7 #include <signal.h>
8 #include <ctype.h>
9 #include <sys/stat.h>
10 #include <unistd.h>
11 #include <inttypes.h>
12 #include <time.h>
13 #include <sys/wait.h>
14 #include <fnmatch.h>
15 #include <limits.h>
16
17
18 #include "gcc-compat.h"
19 #include "cmdline.h"
20 #include "log.h"
21 #include "string.h"
22 #include "error.h"
23 #include "fd.h"
24 #include "exec.h"
25 #include "daemon.h"
26
27
28 struct gengetopt_args_info conf;
29 char *dss_error_txt = NULL;
30 static FILE *logfile;
31
32 DEFINE_DSS_ERRLIST;
33
34
35 /* a litte cpp magic helps to DRY */
36 #define COMMANDS \
37         COMMAND(ls) \
38         COMMAND(create) \
39         COMMAND(prune) \
40         COMMAND(run)
41 #define COMMAND(x) int com_ ##x(void);
42 COMMANDS
43 #undef COMMAND
44 #define COMMAND(x) if (conf.x ##_given) return com_ ##x();
45 int call_command_handler(void)
46 {
47         COMMANDS
48         DSS_EMERG_LOG("BUG: did not find command handler\n");
49         exit(EXIT_FAILURE);
50 }
51 #undef COMMAND
52 #undef COMMANDS
53
54 /*
55  * complete, not being deleted: 1204565370-1204565371.Sun_Mar_02_2008_14_33-Sun_Mar_02_2008_14_43
56  * complete, being deleted: 1204565370-1204565371.being_deleted
57  * incomplete, not being deleted: 1204565370-incomplete
58  * incomplete, being deleted: 1204565370-incomplete.being_deleted
59  */
60 enum snapshot_status_flags {
61         SS_COMPLETE = 1,
62         SS_BEING_DELETED = 2,
63 };
64
65 struct snapshot {
66         char *name;
67         int64_t creation_time;
68         int64_t completion_time;
69         enum snapshot_status_flags flags;
70         unsigned interval;
71 };
72
73 __printf_2_3 void dss_log(int ll, const char* fmt,...)
74 {
75         va_list argp;
76         FILE *outfd;
77         struct tm *tm;
78         time_t t1;
79         char str[255] = "";
80
81         if (ll < conf.loglevel_arg)
82                 return;
83         outfd = logfile? logfile : stderr;
84         time(&t1);
85         tm = localtime(&t1);
86         strftime(str, sizeof(str), "%b %d %H:%M:%S", tm);
87         fprintf(outfd, "%s ", str);
88         if (conf.loglevel_arg <= INFO)
89                 fprintf(outfd, "%i: ", ll);
90         va_start(argp, fmt);
91         vfprintf(outfd, fmt, argp);
92         va_end(argp);
93 }
94
95 __printf_1_2 void msg(const char* fmt,...)
96 {
97         FILE *outfd = conf.daemon_given? logfile : stdout;
98         va_list argp;
99         va_start(argp, fmt);
100         vfprintf(outfd, fmt, argp);
101         va_end(argp);
102 }
103
104 int is_snapshot(const char *dirname, int64_t now, struct snapshot *s)
105 {
106         int i, ret;
107         char *dash, *dot, *tmp;
108         int64_t num;
109
110         assert(dirname);
111         dash = strchr(dirname, '-');
112         if (!dash || !dash[1] || dash == dirname)
113                 return 0;
114         for (i = 0; dirname[i] != '-'; i++)
115                 if (!isdigit(dirname[i]))
116                         return 0;
117         tmp = dss_strdup(dirname);
118         tmp[i] = '\0';
119         ret = dss_atoi64(tmp, &num);
120         free(tmp);
121         if (ret < 0) {
122                 free(dss_error_txt);
123                 return 0;
124         }
125         assert(num >= 0);
126         if (num > now)
127                 return 0;
128         s->creation_time = num;
129         //DSS_DEBUG_LOG("%s start time: %lli\n", dirname, (long long)s->creation_time);
130         s->interval = (long long) ((now - s->creation_time)
131                 / conf.unit_interval_arg / 24 / 3600);
132         if (!strcmp(dash + 1, "incomplete")) {
133                 s->completion_time = -1;
134                 s->flags = 0; /* neither complete, nor being deleted */
135                 goto success;
136         }
137         if (!strcmp(dash + 1, "incomplete.being_deleted")) {
138                 s->completion_time = -1;
139                 s->flags = SS_BEING_DELETED; /* mot cpmplete, being deleted */
140                 goto success;
141         }
142         tmp = dash + 1;
143         dot = strchr(tmp, '.');
144         if (!dot || !dot[1] || dot == tmp)
145                 return 0;
146         for (i = 0; tmp[i] != '.'; i++)
147                 if (!isdigit(tmp[i]))
148                         return 0;
149         tmp = dss_strdup(dash + 1);
150         tmp[i] = '\0';
151         ret = dss_atoi64(tmp, &num);
152         free(tmp);
153         if (ret < 0) {
154                 free(dss_error_txt);
155                 return 0;
156         }
157         if (num > now)
158                 return 0;
159         s->completion_time = num;
160         s->flags = SS_COMPLETE;
161         if (strcmp(dot + 1, "being_deleted"))
162                 s->flags |= SS_BEING_DELETED;
163 success:
164         s->name = dss_strdup(dirname);
165         return 1;
166 }
167
168 int64_t get_current_time(void)
169 {
170         time_t now;
171         time(&now);
172         DSS_DEBUG_LOG("now: %lli\n", (long long) now);
173         return (int64_t)now;
174 }
175
176 char *incomplete_name(int64_t start)
177 {
178         return make_message("%lli-incomplete", (long long)start);
179 }
180
181 char *being_deleted_name(struct snapshot *s)
182 {
183         if (s->flags & SS_COMPLETE)
184                 return make_message("%lli-%lli.being_deleted",
185                         (long long)s->creation_time,
186                         (long long)s->completion_time);
187         return make_message("%lli-incomplete.being_deleted",
188                 (long long)s->creation_time);
189 }
190
191 int complete_name(int64_t start, int64_t end, char **result)
192 {
193         struct tm start_tm, end_tm;
194         time_t *start_seconds = (time_t *) (uint64_t *)&start; /* STFU, gcc */
195         time_t *end_seconds = (time_t *) (uint64_t *)&end; /* STFU, gcc */
196         char start_str[200], end_str[200];
197
198         if (!localtime_r(start_seconds, &start_tm)) {
199                 make_err_msg("%lli", (long long)start);
200                 return -E_LOCALTIME;
201         }
202         if (!localtime_r(end_seconds, &end_tm)) {
203                 make_err_msg("%lli", (long long)end);
204                 return -E_LOCALTIME;
205         }
206         if (!strftime(start_str, sizeof(start_str), "%a_%b_%d_%Y_%H_%M_%S", &start_tm)) {
207                 make_err_msg("%lli", (long long)start);
208                 return -E_STRFTIME;
209         }
210         if (!strftime(end_str, sizeof(end_str), "%a_%b_%d_%Y_%H_%M_%S", &end_tm)) {
211                 make_err_msg("%lli", (long long)end);
212                 return -E_STRFTIME;
213         }
214         *result = make_message("%lli-%lli.%s-%s", (long long) start, (long long) end,
215                 start_str, end_str);
216         return 1;
217 }
218
219 struct snapshot_list {
220         int64_t now;
221         unsigned num_snapshots;
222         unsigned array_size;
223         struct snapshot **snapshots;
224         /**
225          * Array of size num_intervals + 1
226          *
227          * It contains the number of snapshots in each interval. interval_count[num_intervals]
228          * is the number of snapshots which belong to any interval greater than num_intervals.
229          */
230         unsigned *interval_count;
231 };
232
233 #define FOR_EACH_SNAPSHOT(s, i, sl) \
234         for ((i) = 0; (i) < (sl)->num_snapshots && ((s) = (sl)->snapshots[(i)]); (i)++)
235
236
237
238 #define NUM_COMPARE(x, y) ((int)((x) < (y)) - (int)((x) > (y)))
239
240 static int compare_snapshots(const void *a, const void *b)
241 {
242         struct snapshot *s1 = *(struct snapshot **)a;
243         struct snapshot *s2 = *(struct snapshot **)b;
244         return NUM_COMPARE(s2->creation_time, s1->creation_time);
245 }
246
247 /** Compute the minimum of \a a and \a b. */
248 #define DSS_MIN(a,b) ((a) < (b) ? (a) : (b))
249
250 int add_snapshot(const char *dirname, void *private)
251 {
252         struct snapshot_list *sl = private;
253         struct snapshot s;
254         int ret = is_snapshot(dirname, sl->now, &s);
255
256         if (!ret)
257                 return 1;
258         if (sl->num_snapshots >= sl->array_size) {
259                 sl->array_size = 2 * sl->array_size + 1;
260                 sl->snapshots = dss_realloc(sl->snapshots,
261                         sl->array_size * sizeof(struct snapshot *));
262         }
263         sl->snapshots[sl->num_snapshots] = dss_malloc(sizeof(struct snapshot));
264         *(sl->snapshots[sl->num_snapshots]) = s;
265         sl->interval_count[DSS_MIN(s.interval, conf.num_intervals_arg)]++;
266         sl->num_snapshots++;
267         return 1;
268 }
269
270 void get_snapshot_list(struct snapshot_list *sl)
271 {
272         sl->now = get_current_time();
273         sl->num_snapshots = 0;
274         sl->array_size = 0;
275         sl->snapshots = NULL;
276         sl->interval_count = dss_calloc((conf.num_intervals_arg + 1) * sizeof(unsigned));
277         for_each_subdir(add_snapshot, sl);
278         qsort(sl->snapshots, sl->num_snapshots, sizeof(struct snapshot *),
279                 compare_snapshots);
280 }
281
282 void free_snapshot_list(struct snapshot_list *sl)
283 {
284         int i;
285         struct snapshot *s;
286
287         FOR_EACH_SNAPSHOT(s, i, sl) {
288                 free(s->name);
289                 free(s);
290         }
291         free(sl->interval_count);
292         free(sl->snapshots);
293 }
294
295 /**
296  * Print a log message about the exit status of a child.
297  */
298 void log_termination_msg(pid_t pid, int status)
299 {
300         if (WIFEXITED(status))
301                 DSS_INFO_LOG("child %i exited. Exit status: %i\n", (int)pid,
302                         WEXITSTATUS(status));
303         else if (WIFSIGNALED(status))
304                 DSS_NOTICE_LOG("child %i was killed by signal %i\n", (int)pid,
305                         WTERMSIG(status));
306         else
307                 DSS_WARNING_LOG("child %i terminated abormally\n", (int)pid);
308 }
309
310 int wait_for_process(pid_t pid, int *status)
311 {
312         int ret;
313
314         DSS_DEBUG_LOG("Waiting for process %d to terminate\n", (int)pid);
315         for (;;) {
316                 ret = waitpid(pid, status, 0);
317                 if (ret >= 0 || errno != EINTR)
318                         break;
319         }
320         if (ret < 0) {
321                 ret = -ERRNO_TO_DSS_ERROR(errno);
322                 make_err_msg("failed to wait for process %d", (int)pid);
323         } else
324                 log_termination_msg(pid, *status);
325         return ret;
326 }
327
328 int remove_snapshot(struct snapshot *s, pid_t *pid)
329 {
330         int fds[3] = {0, 0, 0};
331         char *new_name = being_deleted_name(s);
332         int ret = dss_rename(s->name, new_name);
333         char *argv[] = {"rm", "-rf", new_name, NULL};
334
335         if (ret < 0)
336                 goto out;
337         DSS_NOTICE_LOG("removing %s (interval = %i)\n", s->name, s->interval);
338         ret = dss_exec(pid, argv[0], argv, fds);
339 out:
340         free(new_name);
341         return ret;
342 }
343
344 int remove_redundant_snapshot(struct snapshot_list *sl,
345                 int dry_run, pid_t *pid)
346 {
347         int ret, i, interval;
348         struct snapshot *s;
349         unsigned missing = 0;
350
351         DSS_INFO_LOG("looking for intervals containing too many snapshots\n");
352         for (interval = conf.num_intervals_arg - 1; interval >= 0; interval--) {
353                 unsigned keep = 1<<(conf.num_intervals_arg - interval - 1);
354                 unsigned num = sl->interval_count[interval];
355                 struct snapshot *victim = NULL, *prev = NULL;
356                 int64_t score = LONG_MAX;
357
358                 if (keep >= num)
359                         missing += keep - num;
360                 DSS_DEBUG_LOG("interval %i: keep: %u, have: %u, missing: %u\n",
361                         interval, keep, num, missing);
362                 if (keep + missing >= num)
363                         continue;
364                 /* redundant snapshot in this interval, pick snapshot with lowest score */
365                 FOR_EACH_SNAPSHOT(s, i, sl) {
366                         int64_t this_score;
367
368                         DSS_DEBUG_LOG("checking %s\n", s->name);
369                         if (s->interval > interval) {
370                                 prev = s;
371                                 continue;
372                         }
373                         if (s->interval < interval)
374                                 break;
375                         if (!victim) {
376                                 victim = s;
377                                 prev = s;
378                                 continue;
379                         }
380                         assert(prev);
381                         /* check if s is a better victim */
382                         this_score = s->creation_time - prev->creation_time;
383                         assert(this_score >= 0);
384                         DSS_DEBUG_LOG("%s: score %lli\n", s->name, (long long)score);
385                         if (this_score < score) {
386                                 score = this_score;
387                                 victim = s;
388                         }
389                         prev = s;
390                 }
391                 assert(victim);
392                 if (dry_run) {
393                         msg("%s would be removed (interval = %i)\n",
394                                 victim->name, victim->interval);
395                         continue;
396                 }
397                 ret = remove_snapshot(victim, pid);
398                 return ret < 0? ret : 1;
399         }
400         return 0;
401 }
402
403 int remove_old_snapshot(struct snapshot_list *sl, int dry_run, pid_t *pid)
404 {
405         int i, ret;
406         struct snapshot *s;
407
408         DSS_INFO_LOG("looking for snapshots belonging to intervals greater than %d\n",
409                 conf.num_intervals_arg);
410         FOR_EACH_SNAPSHOT(s, i, sl) {
411                 if (s->interval <= conf.num_intervals_arg)
412                         continue;
413                 if (dry_run) {
414                         msg("%s would be removed (interval = %i)\n",
415                                 s->name, s->interval);
416                         continue;
417                 }
418                 ret = remove_snapshot(s, pid);
419                 if (ret < 0)
420                         return ret;
421                 return 1;
422         }
423         return 0;
424 }
425
426 int wait_for_rm_process(pid_t pid)
427 {
428         int status, es, ret = wait_for_process(pid, &status);
429         if (ret < 0)
430                 return ret;
431         if (!WIFEXITED(status)) {
432                 ret = E_INVOLUNTARY_EXIT;
433                 make_err_msg("rm process %d died involuntary", (int)pid);
434                 return ret;
435         }
436         es = WEXITSTATUS(status);
437         if (es) {
438                 ret = -E_BAD_EXIT_CODE;
439                 make_err_msg("rm process %d returned %d", (int)pid, es);
440                 return ret;
441         }
442         return 1;
443 }
444
445 int com_run(void)
446 {
447         return 42;
448 }
449
450 int com_prune(void)
451 {
452         int ret, dry_run = 0;
453         struct snapshot_list sl;
454         pid_t pid;
455
456         for (;;) {
457                 get_snapshot_list(&sl);
458                 ret = remove_old_snapshot(&sl, dry_run, &pid);
459                 free_snapshot_list(&sl);
460                 if (ret < 0)
461                         return ret;
462                 if (!ret)
463                         break;
464                 ret = wait_for_rm_process(pid);
465                 if (ret < 0)
466                         goto out;
467         }
468         for (;;) {
469                 get_snapshot_list(&sl);
470                 ret = remove_redundant_snapshot(&sl, dry_run, &pid);
471                 free_snapshot_list(&sl);
472                 if (ret < 0)
473                         return ret;
474                 if (!ret)
475                         break;
476                 ret = wait_for_rm_process(pid);
477                 if (ret < 0)
478                         goto out;
479         }
480         return 1;
481 out:
482         return ret;
483 }
484
485 struct newest_snapshot_data {
486         char * newest_name;
487         int64_t newest_creation_time;
488         int64_t now;
489 };
490
491 int get_newest_complete(const char *dirname, void *private)
492 {
493         struct newest_snapshot_data *nsd = private;
494         struct snapshot s;
495         int ret = is_snapshot(dirname, nsd->now, &s);
496
497         if (ret <= 0)
498                 return 1;
499         if (s.creation_time < nsd->newest_creation_time)
500                 return 1;
501         nsd->newest_creation_time = s.creation_time;
502         free(nsd->newest_name);
503         nsd->newest_name = s.name;
504         return 1;
505 }
506
507 __malloc char *name_of_newest_complete_snapshot(void)
508 {
509         struct newest_snapshot_data nsd = {
510                 .now = get_current_time(),
511                 .newest_creation_time = -1
512         };
513         for_each_subdir(get_newest_complete, &nsd);
514         return nsd.newest_name;
515 }
516
517 void create_rsync_argv(char ***argv, int64_t *num)
518 {
519         char *logname, *newest = name_of_newest_complete_snapshot();
520         int i = 0, j;
521
522         *argv = dss_malloc((15 + conf.rsync_option_given) * sizeof(char *));
523         (*argv)[i++] = dss_strdup("rsync");
524         (*argv)[i++] = dss_strdup("-aq");
525         (*argv)[i++] = dss_strdup("--delete");
526         for (j = 0; j < conf.rsync_option_given; j++)
527                 (*argv)[i++] = dss_strdup(conf.rsync_option_arg[j]);
528         if (newest) {
529                 DSS_INFO_LOG("using %s as reference snapshot\n", newest);
530                 (*argv)[i++] = make_message("--link-dest=../%s", newest);
531                 free(newest);
532         } else
533                 DSS_INFO_LOG("no previous snapshot found");
534         if (conf.exclude_patterns_given) {
535                 (*argv)[i++] = dss_strdup("--exclude-from");
536                 (*argv)[i++] = dss_strdup(conf.exclude_patterns_arg);
537
538         }
539         logname = dss_logname();
540         if (conf.remote_user_given && !strcmp(conf.remote_user_arg, logname))
541                 (*argv)[i++] = dss_strdup(conf.source_dir_arg);
542         else
543                 (*argv)[i++] = make_message("%s@%s:%s/", conf.remote_user_given?
544                         conf.remote_user_arg : logname,
545                         conf.remote_host_arg, conf.source_dir_arg);
546         free(logname);
547         *num = get_current_time();
548         (*argv)[i++] = incomplete_name(*num);
549         (*argv)[i++] = NULL;
550         for (j = 0; j < i; j++)
551                 DSS_DEBUG_LOG("argv[%d] = %s\n", j, (*argv)[j]);
552 }
553
554 void free_rsync_argv(char **argv)
555 {
556         int i;
557         for (i = 0; argv[i]; i++)
558                 free(argv[i]);
559         free(argv);
560 }
561
562 int create_snapshot(char **argv, pid_t *pid)
563 {
564         int fds[3] = {0, 0, 0};
565
566         return dss_exec(pid, argv[0], argv, fds);
567 }
568
569 int rename_incomplete_snapshot(int64_t start)
570 {
571         char *old_name, *new_name;
572         int ret;
573
574         ret = complete_name(start, get_current_time(), &new_name);
575         if (ret < 0)
576                 return ret;
577         old_name = incomplete_name(start);
578         ret = dss_rename(old_name, new_name);
579         if (ret >= 0)
580                 DSS_NOTICE_LOG("%s -> %s\n", old_name, new_name);
581         free(old_name);
582         free(new_name);
583         return ret;
584 }
585
586 int com_create(void)
587 {
588         int ret, status, es;
589         char **rsync_argv;
590         int64_t snapshot_num;
591         pid_t pid;
592
593         create_rsync_argv(&rsync_argv, &snapshot_num);
594         DSS_NOTICE_LOG("creating snapshot %lli\n", (long long)snapshot_num);
595         ret = create_snapshot(rsync_argv, &pid);
596         if (ret < 0)
597                 goto out;
598         ret = wait_for_process(pid, &status);
599         if (ret < 0)
600                 goto out;
601         if (!WIFEXITED(status)) {
602                 ret = E_INVOLUNTARY_EXIT;
603                 make_err_msg("rsync process %d died involuntary", (int)pid);
604                 goto out;
605         }
606         es = WEXITSTATUS(status);
607         if (es != 0 && es != 23 && es != 24) {
608                 ret = -E_BAD_EXIT_CODE;
609                 make_err_msg("rsync process %d returned %d", (int)pid, es);
610                 goto out;
611         }
612         ret = rename_incomplete_snapshot(snapshot_num);
613 out:
614         free_rsync_argv(rsync_argv);
615         return ret;
616 }
617
618 int com_ls(void)
619 {
620         int i;
621         struct snapshot_list sl;
622         struct snapshot *s;
623         get_snapshot_list(&sl);
624         FOR_EACH_SNAPSHOT(s, i, &sl)
625                 msg("%u\t%s\n", s->interval, s->name);
626         free_snapshot_list(&sl);
627         return 1;
628 }
629
630 /* TODO: Unlink pid file */
631 __noreturn void clean_exit(int status)
632 {
633         //kill(0, SIGTERM);
634         free(dss_error_txt);
635         exit(status);
636 }
637
638 int read_config_file(void)
639 {
640         int ret;
641         char *config_file;
642         struct stat statbuf;
643
644         if (conf.config_file_given)
645                 config_file = dss_strdup(conf.config_file_arg);
646         else {
647                 char *home = get_homedir();
648                 config_file = make_message("%s/.dssrc", home);
649                 free(home);
650         }
651         ret = stat(config_file, &statbuf);
652         if (ret && conf.config_file_given) {
653                 ret = -ERRNO_TO_DSS_ERROR(errno);
654                 make_err_msg("failed to stat config file %s", config_file);
655                 goto out;
656         }
657         if (!ret) {
658                 struct cmdline_parser_params params = {
659                         .override = 0,
660                         .initialize = 0,
661                         .check_required = 0,
662                         .check_ambiguity = 0
663                 };
664                 cmdline_parser_config_file(config_file, &conf, &params);
665         }
666         if (!conf.source_dir_given || !conf.dest_dir_given) {
667                 ret = -E_SYNTAX;
668                 make_err_msg("you need to specify both source_dir and dest_dir");
669                 goto out;
670         }
671         ret = 1;
672 out:
673         free(config_file);
674         return ret;
675 }
676
677 int check_config(void)
678 {
679         if (conf.unit_interval_arg <= 0) {
680                 make_err_msg("bad unit interval: %i", conf.unit_interval_arg);
681                 return -E_INVALID_NUMBER;
682         }
683         DSS_DEBUG_LOG("unit interval: %i day(s)\n", conf.unit_interval_arg);
684         if (conf.num_intervals_arg <= 0) {
685                 make_err_msg("bad number of intervals  %i", conf.num_intervals_arg);
686                 return -E_INVALID_NUMBER;
687         }
688         DSS_DEBUG_LOG("number of intervals: %i\n", conf.num_intervals_arg);
689         return 1;
690 }
691
692 int main(int argc, char **argv)
693 {
694         int ret;
695
696         cmdline_parser(argc, argv, &conf); /* aborts on errors */
697         if (conf.inputs_num) {
698                 ret = -E_SYNTAX;
699                 make_err_msg("additional non-options given");
700                 goto out;
701         }
702         ret = read_config_file();
703         if (ret < 0)
704                 goto out;
705         ret = check_config();
706         if (ret < 0)
707                 goto out;
708         if (conf.logfile_given) {
709                 logfile = open_log(conf.logfile_arg);
710                 log_welcome(conf.loglevel_arg);
711         }
712         if (conf.daemon_given)
713                 daemon_init();
714         ret = dss_chdir(conf.dest_dir_arg);
715         if (ret < 0)
716                 goto out;
717         ret = call_command_handler();
718 out:
719         if (ret < 0)
720                 log_err_msg(EMERG, -ret);
721         clean_exit(ret >= 0? EXIT_SUCCESS : EXIT_FAILURE);
722 }