]> git.tuebingen.mpg.de Git - dss.git/blob - dss.c
0913301386fefacfda2bcef9016cce9513983ebd
[dss.git] / dss.c
1 #include <string.h>
2 #include <stdlib.h>
3 #include <stdarg.h>
4 #include <assert.h>
5 #include <errno.h>
6 #include <sys/types.h>
7 #include <signal.h>
8 #include <ctype.h>
9 #include <sys/stat.h>
10 #include <unistd.h>
11 #include <inttypes.h>
12 #include <time.h>
13 #include <sys/wait.h>
14 #include <fnmatch.h>
15 #include <limits.h>
16
17
18 #include "gcc-compat.h"
19 #include "cmdline.h"
20 #include "log.h"
21 #include "string.h"
22 #include "error.h"
23 #include "fd.h"
24 #include "exec.h"
25
26
27 struct gengetopt_args_info conf;
28 char *dss_error_txt = NULL;
29
30 DEFINE_DSS_ERRLIST;
31
32 /** Defines one dss command. */
33 struct server_command {
34         /** The name of the command. */
35         const char *name;
36         /** Pointer to the function that handles the command. */
37         int (*handler)(int, char * const * const);
38 };
39
40 /* a litte cpp magic helps to DRY */
41 #define SERVER_COMMANDS \
42         SERVER_COMMAND(ls) \
43         SERVER_COMMAND(create) \
44         SERVER_COMMAND(prune) \
45         SERVER_COMMAND(daemon)
46 #define SERVER_COMMAND(x) int com_ ##x(int, char * const * const);
47 SERVER_COMMANDS
48 #undef SERVER_COMMAND
49 #define SERVER_COMMAND(x) {.name = #x, .handler = com_ ## x},
50 static struct server_command command_list[] = {
51         SERVER_COMMANDS
52         {.name = NULL, .handler = NULL}
53 };
54 #undef SERVER_COMMAND
55
56 /*
57  * complete, not being deleted: 1204565370-1204565371.Sun_Mar_02_2008_14_33-Sun_Mar_02_2008_14_43
58  * complete, being deleted: 1204565370-1204565371.being_deleted
59  * incomplete, not being deleted: 1204565370-incomplete
60  * incomplete, being deleted: 1204565370-incomplete.being_deleted
61  */
62 enum snapshot_status_flags {
63         SS_COMPLETE = 1,
64         SS_BEING_DELETED = 2,
65 };
66
67 struct snapshot {
68         char *name;
69         int64_t creation_time;
70         int64_t completion_time;
71         enum snapshot_status_flags flags;
72         unsigned interval;
73 };
74
75 int is_snapshot(const char *dirname, int64_t now, struct snapshot *s)
76 {
77         int i, ret;
78         char *dash, *dot, *tmp;
79         int64_t num;
80
81         assert(dirname);
82         dash = strchr(dirname, '-');
83         if (!dash || !dash[1] || dash == dirname)
84                 return 0;
85         for (i = 0; dirname[i] != '-'; i++)
86                 if (!isdigit(dirname[i]))
87                         return 0;
88         tmp = dss_strdup(dirname);
89         tmp[i] = '\0';
90         ret = dss_atoi64(tmp, &num);
91         free(tmp);
92         if (ret < 0) {
93                 free(dss_error_txt);
94                 return 0;
95         }
96         assert(num >= 0);
97         if (num > now)
98                 return 0;
99         s->creation_time = num;
100         //DSS_DEBUG_LOG("%s start time: %lli\n", dirname, (long long)s->creation_time);
101         s->interval = (long long) ((now - s->creation_time)
102                 / conf.unit_interval_arg / 24 / 3600);
103         if (!strcmp(dash + 1, "incomplete")) {
104                 s->completion_time = -1;
105                 s->flags = 0; /* neither complete, nor being deleted */
106                 goto success;
107         }
108         if (!strcmp(dash + 1, "incomplete.being_deleted")) {
109                 s->completion_time = -1;
110                 s->flags = SS_BEING_DELETED; /* mot cpmplete, being deleted */
111                 goto success;
112         }
113         tmp = dash + 1;
114         dot = strchr(tmp, '.');
115         if (!dot || !dot[1] || dot == tmp)
116                 return 0;
117         for (i = 0; tmp[i] != '.'; i++)
118                 if (!isdigit(tmp[i]))
119                         return 0;
120         tmp = dss_strdup(dash + 1);
121         tmp[i] = '\0';
122         ret = dss_atoi64(tmp, &num);
123         free(tmp);
124         if (ret < 0) {
125                 free(dss_error_txt);
126                 return 0;
127         }
128         if (num > now)
129                 return 0;
130         s->completion_time = num;
131         s->flags = SS_COMPLETE;
132         if (strcmp(dot + 1, "being_deleted"))
133                 s->flags |= SS_BEING_DELETED;
134 success:
135         s->name = dss_strdup(dirname);
136         return 1;
137 }
138
139 int64_t get_current_time(void)
140 {
141         time_t now;
142         time(&now);
143         DSS_DEBUG_LOG("now: %lli\n", (long long) now);
144         return (int64_t)now;
145 }
146
147 char *incomplete_name(int64_t start)
148 {
149         return make_message("%lli-incomplete", (long long)start);
150 }
151
152 char *being_deleted_name(struct snapshot *s)
153 {
154         if (s->flags & SS_COMPLETE)
155                 return make_message("%lli-%lli.being_deleted",
156                         (long long)s->creation_time,
157                         (long long)s->completion_time);
158         return make_message("%lli-incomplete.being_deleted",
159                 (long long)s->creation_time);
160 }
161
162 int complete_name(int64_t start, int64_t end, char **result)
163 {
164         struct tm start_tm, end_tm;
165         time_t *start_seconds = (time_t *) (uint64_t *)&start; /* STFU, gcc */
166         time_t *end_seconds = (time_t *) (uint64_t *)&end; /* STFU, gcc */
167         char start_str[200], end_str[200];
168
169         if (!localtime_r(start_seconds, &start_tm)) {
170                 make_err_msg("%lli", (long long)start);
171                 return -E_LOCALTIME;
172         }
173         if (!localtime_r(end_seconds, &end_tm)) {
174                 make_err_msg("%lli", (long long)end);
175                 return -E_LOCALTIME;
176         }
177         if (!strftime(start_str, sizeof(start_str), "%a_%b_%d_%Y_%H_%M_%S", &start_tm)) {
178                 make_err_msg("%lli", (long long)start);
179                 return -E_STRFTIME;
180         }
181         if (!strftime(end_str, sizeof(end_str), "%a_%b_%d_%Y_%H_%M_%S", &end_tm)) {
182                 make_err_msg("%lli", (long long)end);
183                 return -E_STRFTIME;
184         }
185         *result = make_message("%lli-%lli.%s-%s", (long long) start, (long long) end,
186                 start_str, end_str);
187         return 1;
188 }
189
190 struct snapshot_list {
191         int64_t now;
192         unsigned num_snapshots;
193         unsigned array_size;
194         struct snapshot **snapshots;
195         /**
196          * Array of size num_intervals + 1
197          *
198          * It contains the number of snapshots in each interval. interval_count[num_intervals]
199          * is the number of snapshots which belong to any interval greater than num_intervals.
200          */
201         unsigned *interval_count;
202 };
203
204 #define FOR_EACH_SNAPSHOT(s, i, sl) \
205         for ((i) = 0; (i) < (sl)->num_snapshots && ((s) = (sl)->snapshots[(i)]); (i)++)
206
207
208
209 #define NUM_COMPARE(x, y) ((int)((x) < (y)) - (int)((x) > (y)))
210
211 static int compare_snapshots(const void *a, const void *b)
212 {
213         struct snapshot *s1 = *(struct snapshot **)a;
214         struct snapshot *s2 = *(struct snapshot **)b;
215         return NUM_COMPARE(s2->creation_time, s1->creation_time);
216 }
217
218 /** Compute the minimum of \a a and \a b. */
219 #define DSS_MIN(a,b) ((a) < (b) ? (a) : (b))
220
221 int add_snapshot(const char *dirname, void *private)
222 {
223         struct snapshot_list *sl = private;
224         struct snapshot s;
225         int ret = is_snapshot(dirname, sl->now, &s);
226
227         if (!ret)
228                 return 1;
229         if (sl->num_snapshots >= sl->array_size) {
230                 sl->array_size = 2 * sl->array_size + 1;
231                 sl->snapshots = dss_realloc(sl->snapshots,
232                         sl->array_size * sizeof(struct snapshot *));
233         }
234         sl->snapshots[sl->num_snapshots] = dss_malloc(sizeof(struct snapshot));
235         *(sl->snapshots[sl->num_snapshots]) = s;
236         sl->interval_count[DSS_MIN(s.interval, conf.num_intervals_arg)]++;
237         sl->num_snapshots++;
238         return 1;
239 }
240
241 void get_snapshot_list(struct snapshot_list *sl)
242 {
243         sl->now = get_current_time();
244         sl->num_snapshots = 0;
245         sl->array_size = 0;
246         sl->snapshots = NULL;
247         sl->interval_count = dss_calloc((conf.num_intervals_arg + 1) * sizeof(unsigned));
248         for_each_subdir(add_snapshot, sl);
249         qsort(sl->snapshots, sl->num_snapshots, sizeof(struct snapshot *),
250                 compare_snapshots);
251 }
252
253 void free_snapshot_list(struct snapshot_list *sl)
254 {
255         int i;
256         struct snapshot *s;
257
258         FOR_EACH_SNAPSHOT(s, i, sl) {
259                 free(s->name);
260                 free(s);
261         }
262         free(sl->interval_count);
263         free(sl->snapshots);
264 }
265
266 /**
267  * Print a log message about the exit status of a child.
268  */
269 void log_termination_msg(pid_t pid, int status)
270 {
271         if (WIFEXITED(status))
272                 DSS_INFO_LOG("child %i exited. Exit status: %i\n", (int)pid,
273                         WEXITSTATUS(status));
274         else if (WIFSIGNALED(status))
275                 DSS_NOTICE_LOG("child %i was killed by signal %i\n", (int)pid,
276                         WTERMSIG(status));
277         else
278                 DSS_WARNING_LOG("child %i terminated abormally\n", (int)pid);
279 }
280
281 int wait_for_process(pid_t pid, int *status)
282 {
283         int ret;
284
285         DSS_DEBUG_LOG("Waiting for process %d to terminate\n", (int)pid);
286         for (;;) {
287                 ret = waitpid(pid, status, 0);
288                 if (ret >= 0 || errno != EINTR)
289                         break;
290         }
291         if (ret < 0) {
292                 ret = -ERRNO_TO_DSS_ERROR(errno);
293                 make_err_msg("failed to wait for process %d", (int)pid);
294         } else
295                 log_termination_msg(pid, *status);
296         return ret;
297 }
298
299 int remove_snapshot(struct snapshot *s, pid_t *pid)
300 {
301         int fds[3] = {0, 0, 0};
302         char *new_name = being_deleted_name(s);
303         int ret = dss_rename(s->name, new_name);
304         char *argv[] = {"rm", "-rf", new_name, NULL};
305
306         if (ret < 0)
307                 goto out;
308         DSS_NOTICE_LOG("removing %s (interval = %i)\n", s->name, s->interval);
309         ret = dss_exec(pid, argv[0], argv, fds);
310 out:
311         free(new_name);
312         return ret;
313 }
314
315 int remove_redundant_snapshot(struct snapshot_list *sl,
316                 int dry_run, pid_t *pid)
317 {
318         int ret, i, interval;
319         struct snapshot *s;
320         unsigned missing = 0;
321
322         DSS_INFO_LOG("looking for intervals containing too many snapshots\n");
323         for (interval = conf.num_intervals_arg - 1; interval >= 0; interval--) {
324                 unsigned keep = 1<<(conf.num_intervals_arg - interval - 1);
325                 unsigned num = sl->interval_count[interval];
326                 struct snapshot *victim = NULL, *prev = NULL;
327                 int64_t score = LONG_MAX;
328
329                 if (keep >= num)
330                         missing += keep - num;
331                 DSS_DEBUG_LOG("interval %i: keep: %u, have: %u, missing: %u\n",
332                         interval, keep, num, missing);
333                 if (keep + missing >= num)
334                         continue;
335                 /* redundant snapshot in this interval, pick snapshot with lowest score */
336                 FOR_EACH_SNAPSHOT(s, i, sl) {
337                         int64_t this_score;
338
339                         DSS_DEBUG_LOG("checking %s\n", s->name);
340                         if (s->interval > interval) {
341                                 prev = s;
342                                 continue;
343                         }
344                         if (s->interval < interval)
345                                 break;
346                         if (!victim) {
347                                 victim = s;
348                                 prev = s;
349                                 continue;
350                         }
351                         assert(prev);
352                         /* check if s is a better victim */
353                         this_score = s->creation_time - prev->creation_time;
354                         assert(this_score >= 0);
355                         DSS_DEBUG_LOG("%s: score %lli\n", s->name, (long long)score);
356                         if (this_score < score) {
357                                 score = this_score;
358                                 victim = s;
359                         }
360                         prev = s;
361                 }
362                 assert(victim);
363                 if (dry_run) {
364                         printf("%s would be removed (interval = %i)\n",
365                                 victim->name, victim->interval);
366                         continue;
367                 }
368                 ret = remove_snapshot(victim, pid);
369                 return ret < 0? ret : 1;
370         }
371         return 0;
372 }
373
374 int remove_old_snapshot(struct snapshot_list *sl, int dry_run, pid_t *pid)
375 {
376         int i, ret;
377         struct snapshot *s;
378
379         DSS_INFO_LOG("looking for snapshots belonging to intervals greater than %d\n",
380                 conf.num_intervals_arg);
381         FOR_EACH_SNAPSHOT(s, i, sl) {
382                 if (s->interval <= conf.num_intervals_arg)
383                         continue;
384                 if (dry_run) {
385                         printf("%s would be removed (interval = %i)\n",
386                                 s->name, s->interval);
387                         continue;
388                 }
389                 ret = remove_snapshot(s, pid);
390                 if (ret < 0)
391                         return ret;
392                 return 1;
393         }
394         return 0;
395 }
396
397 int wait_for_rm_process(pid_t pid)
398 {
399         int status, es, ret = wait_for_process(pid, &status);
400         if (ret < 0)
401                 return ret;
402         if (!WIFEXITED(status)) {
403                 ret = E_INVOLUNTARY_EXIT;
404                 make_err_msg("rm process %d died involuntary", (int)pid);
405                 return ret;
406         }
407         es = WEXITSTATUS(status);
408         if (es) {
409                 ret = -E_BAD_EXIT_CODE;
410                 make_err_msg("rm process %d returned %d", (int)pid, es);
411                 return ret;
412         }
413         return 1;
414 }
415
416 int com_prune(int argc, char * const * argv)
417 {
418         int ret, dry_run = 0;
419         struct snapshot_list sl;
420         pid_t pid;
421
422         if (argc > 2) {
423                 make_err_msg("too many arguments");
424                 return -E_SYNTAX;
425         }
426         if (argc == 2) {
427                 if (strcmp(argv[1], "-d")) {
428                         make_err_msg("%s", argv[1]);
429                         return -E_SYNTAX;
430                 }
431                 dry_run = 1;
432         }
433         for (;;) {
434                 get_snapshot_list(&sl);
435                 ret = remove_old_snapshot(&sl, dry_run, &pid);
436                 free_snapshot_list(&sl);
437                 if (ret < 0)
438                         return ret;
439                 if (!ret)
440                         break;
441                 ret = wait_for_rm_process(pid);
442                 if (ret < 0)
443                         goto out;
444         }
445         for (;;) {
446                 get_snapshot_list(&sl);
447                 ret = remove_redundant_snapshot(&sl, dry_run, &pid);
448                 free_snapshot_list(&sl);
449                 if (ret < 0)
450                         return ret;
451                 if (!ret)
452                         break;
453                 ret = wait_for_rm_process(pid);
454                 if (ret < 0)
455                         goto out;
456         }
457         return 1;
458 out:
459         return ret;
460 }
461
462 struct newest_snapshot_data {
463         char * newest_name;
464         int64_t newest_creation_time;
465         int64_t now;
466 };
467
468 int get_newest_complete(const char *dirname, void *private)
469 {
470         struct newest_snapshot_data *nsd = private;
471         struct snapshot s;
472         int ret = is_snapshot(dirname, nsd->now, &s);
473
474         if (ret <= 0)
475                 return 1;
476         if (s.creation_time < nsd->newest_creation_time)
477                 return 1;
478         nsd->newest_creation_time = s.creation_time;
479         free(nsd->newest_name);
480         nsd->newest_name = s.name;
481         return 1;
482 }
483
484 __malloc char *name_of_newest_complete_snapshot(void)
485 {
486         struct newest_snapshot_data nsd = {
487                 .now = get_current_time(),
488                 .newest_creation_time = -1
489         };
490         for_each_subdir(get_newest_complete, &nsd);
491         return nsd.newest_name;
492 }
493
494 void create_rsync_argv(char ***argv, int64_t *num)
495 {
496         char *logname, *newest = name_of_newest_complete_snapshot();
497         int i = 0, j;
498
499         *argv = dss_malloc((15 + conf.rsync_option_given) * sizeof(char *));
500         (*argv)[i++] = dss_strdup("rsync");
501         (*argv)[i++] = dss_strdup("-aq");
502         (*argv)[i++] = dss_strdup("--delete");
503         for (j = 0; j < conf.rsync_option_given; j++)
504                 (*argv)[i++] = dss_strdup(conf.rsync_option_arg[j]);
505         if (newest) {
506                 DSS_INFO_LOG("using %s as reference snapshot\n", newest);
507                 (*argv)[i++] = make_message("--link-dest=../%s", newest);
508                 free(newest);
509         } else
510                 DSS_INFO_LOG("no previous snapshot found");
511         if (conf.exclude_patterns_given) {
512                 (*argv)[i++] = dss_strdup("--exclude-from");
513                 (*argv)[i++] = dss_strdup(conf.exclude_patterns_arg);
514
515         }
516         logname = dss_logname();
517         if (conf.remote_user_given && !strcmp(conf.remote_user_arg, logname))
518                 (*argv)[i++] = dss_strdup(conf.source_dir_arg);
519         else
520                 (*argv)[i++] = make_message("%s@%s:%s/", conf.remote_user_given?
521                         conf.remote_user_arg : logname,
522                         conf.remote_host_arg, conf.source_dir_arg);
523         free(logname);
524         *num = get_current_time();
525         (*argv)[i++] = incomplete_name(*num);
526         (*argv)[i++] = NULL;
527         for (j = 0; j < i; j++)
528                 DSS_DEBUG_LOG("argv[%d] = %s\n", j, (*argv)[j]);
529 }
530
531 void free_rsync_argv(char **argv)
532 {
533         int i;
534         for (i = 0; argv[i]; i++)
535                 free(argv[i]);
536         free(argv);
537 }
538
539 int create_snapshot(char **argv, pid_t *pid)
540 {
541         int fds[3] = {0, 0, 0};
542
543         return dss_exec(pid, argv[0], argv, fds);
544 }
545
546 int rename_incomplete_snapshot(int64_t start)
547 {
548         char *old_name, *new_name;
549         int ret;
550
551         ret = complete_name(start, get_current_time(), &new_name);
552         if (ret < 0)
553                 return ret;
554         old_name = incomplete_name(start);
555         ret = dss_rename(old_name, new_name);
556         if (ret >= 0)
557                 DSS_NOTICE_LOG("%s -> %s\n", old_name, new_name);
558         free(old_name);
559         free(new_name);
560         return ret;
561 }
562
563 int com_create(int argc, __a_unused char * const * argv)
564 {
565         int ret, status, es;
566         char **rsync_argv;
567         int64_t snapshot_num;
568         pid_t pid;
569
570         if (argc != 1) {
571                 ret = -E_SYNTAX;
572                 make_err_msg("create: no args expected, %d given", argc - 1);
573                 return ret;
574         }
575         create_rsync_argv(&rsync_argv, &snapshot_num);
576         DSS_NOTICE_LOG("creating snapshot %lli\n", (long long)snapshot_num);
577         ret = create_snapshot(rsync_argv, &pid);
578         if (ret < 0)
579                 goto out;
580         ret = wait_for_process(pid, &status);
581         if (ret < 0)
582                 goto out;
583         if (!WIFEXITED(status)) {
584                 ret = E_INVOLUNTARY_EXIT;
585                 make_err_msg("rsync process %d died involuntary", (int)pid);
586                 goto out;
587         }
588         es = WEXITSTATUS(status);
589         if (es != 0 && es != 23 && es != 24) {
590                 ret = -E_BAD_EXIT_CODE;
591                 make_err_msg("rsync process %d returned %d", (int)pid, es);
592                 goto out;
593         }
594         ret = rename_incomplete_snapshot(snapshot_num);
595 out:
596         free_rsync_argv(rsync_argv);
597         return ret;
598 }
599
600 int com_ls(int argc, __a_unused char * const * argv)
601 {
602         int i, ret;
603         struct snapshot_list sl;
604         struct snapshot *s;
605         if (argc != 1) {
606                 ret = -E_SYNTAX;
607                 make_err_msg("ls: no args expected, %d given", argc - 1);
608                 return ret;
609         }
610         get_snapshot_list(&sl);
611         FOR_EACH_SNAPSHOT(s, i, &sl)
612                 printf("%u\t%s\n", s->interval, s->name);
613         free_snapshot_list(&sl);
614         return 1;
615 }
616
617 /* TODO: Unlink pid file */
618 __noreturn void clean_exit(int status)
619 {
620         //kill(0, SIGTERM);
621         free(dss_error_txt);
622         exit(status);
623 }
624
625 __printf_2_3 void dss_log(int ll, const char* fmt,...)
626 {
627         va_list argp;
628         if (ll < conf.loglevel_arg)
629                 return;
630         va_start(argp, fmt);
631         vfprintf(stderr, fmt, argp);
632         va_end(argp);
633 }
634
635 int read_config_file(void)
636 {
637         int ret;
638         char *config_file;
639         struct stat statbuf;
640
641         if (conf.config_file_given)
642                 config_file = dss_strdup(conf.config_file_arg);
643         else {
644                 char *home = get_homedir();
645                 config_file = make_message("%s/.dssrc", home);
646                 free(home);
647         }
648         ret = stat(config_file, &statbuf);
649         if (ret && conf.config_file_given) {
650                 ret = -ERRNO_TO_DSS_ERROR(errno);
651                 make_err_msg("failed to stat config file %s", config_file);
652                 goto out;
653         }
654         if (!ret) {
655                 struct cmdline_parser_params params = {
656                         .override = 0,
657                         .initialize = 0,
658                         .check_required = 0,
659                         .check_ambiguity = 0
660                 };
661                 cmdline_parser_config_file(config_file, &conf, &params);
662         }
663         if (!conf.source_dir_given || !conf.dest_dir_given) {
664                 ret = -E_SYNTAX;
665                 make_err_msg("you need to specify both source_dir and dest_dir");
666                 goto out;
667         }
668         ret = 1;
669 out:
670         free(config_file);
671         return ret;
672 }
673
674 int check_config(void)
675 {
676         if (conf.unit_interval_arg <= 0) {
677                 make_err_msg("bad unit interval: %i", conf.unit_interval_arg);
678                 return -E_INVALID_NUMBER;
679         }
680         DSS_DEBUG_LOG("unit interval: %i day(s)\n", conf.unit_interval_arg);
681         if (conf.num_intervals_arg <= 0) {
682                 make_err_msg("bad number of intervals  %i", conf.num_intervals_arg);
683                 return -E_INVALID_NUMBER;
684         }
685         DSS_DEBUG_LOG("number of intervals: %i\n", conf.num_intervals_arg);
686         return 1;
687 }
688
689 int main(int argc, char **argv)
690 {
691         int i, ret;
692
693         cmdline_parser(argc, argv, &conf); /* aborts on errors */
694         if (!conf.inputs_num) {
695                 ret = -E_SYNTAX;
696                 make_err_msg("no command given");
697                 goto out;
698         }
699         ret = read_config_file();
700         if (ret < 0)
701                 goto out;
702         ret = check_config();
703         if (ret < 0)
704                 goto out;
705         ret = dss_chdir(conf.dest_dir_arg);
706         if (ret < 0)
707                 goto out;
708         for (i = 0; command_list[i].name; i++) {
709                 if (strcmp(command_list[i].name, conf.inputs[0]))
710                         continue;
711                 ret = command_list[i].handler(conf.inputs_num, conf.inputs);
712                 goto out;
713         }
714         ret = -E_INVALID_COMMAND;
715         make_err_msg("%s", conf.inputs[0]);
716 out:
717         if (ret < 0)
718                 log_err_msg(EMERG, -ret);
719         clean_exit(ret >= 0? EXIT_SUCCESS : EXIT_FAILURE);
720 }