19 #include "gcc-compat.h"
32 struct gengetopt_args_info conf;
33 char *dss_error_txt = NULL;
35 static int signal_pipe;
37 /** Process id of current rsync process. */
38 static pid_t rsync_pid;
39 /** Whether the rsync process is currently stopped */
40 static int rsync_stopped;
41 /** Process id of current rm process. */
43 /** When the next snapshot is due. */
44 struct timeval next_snapshot_time;
45 /* Creation time of the snapshot currently being created. */
46 int64_t current_snapshot_creation_time;
52 /* a litte cpp magic helps to DRY */
58 #define COMMAND(x) int com_ ##x(void);
61 #define COMMAND(x) if (conf.x ##_given) return com_ ##x();
62 int call_command_handler(void)
65 DSS_EMERG_LOG("BUG: did not find command handler\n");
72 * complete, not being deleted: 1204565370-1204565371.Sun_Mar_02_2008_14_33-Sun_Mar_02_2008_14_43
73 * complete, being deleted: 1204565370-1204565371.being_deleted
74 * incomplete, not being deleted: 1204565370-incomplete
75 * incomplete, being deleted: 1204565370-incomplete.being_deleted
77 enum snapshot_status_flags {
78 /** The rsync process terminated successfully. */
80 /** The rm process is running to remove this snapshot. */
86 int64_t creation_time;
87 int64_t completion_time;
88 enum snapshot_status_flags flags;
93 * An edge snapshot is either the oldest one or the newest one.
95 * We need to find either of them occasionally: The create code
96 * needs to know the newest snapshot because that is the one
97 * used as the link destination dir. The pruning code needs to
98 * find the oldest one in case disk space becomes low.
100 struct edge_snapshot_data {
102 struct snapshot snap;
105 __printf_2_3 void dss_log(int ll, const char* fmt,...)
113 if (ll < conf.loglevel_arg)
115 outfd = logfile? logfile : stderr;
118 strftime(str, sizeof(str), "%b %d %H:%M:%S", tm);
119 fprintf(outfd, "%s ", str);
120 if (conf.loglevel_arg <= INFO)
121 fprintf(outfd, "%i: ", ll);
123 vfprintf(outfd, fmt, argp);
128 * Print a message either to stdout or to the log file.
130 __printf_1_2 void dss_msg(const char* fmt,...)
132 FILE *outfd = conf.daemon_given? logfile : stdout;
135 vfprintf(outfd, fmt, argp);
140 * Return the desired number of snapshots of an interval.
142 unsigned num_snapshots(int interval)
146 assert(interval >= 0);
148 if (interval >= conf.num_intervals_arg)
150 n = conf.num_intervals_arg - interval - 1;
154 /* return: Whether dirname is a snapshot directory (0: no, 1: yes) */
155 int is_snapshot(const char *dirname, int64_t now, struct snapshot *s)
158 char *dash, *dot, *tmp;
162 dash = strchr(dirname, '-');
163 if (!dash || !dash[1] || dash == dirname)
165 for (i = 0; dirname[i] != '-'; i++)
166 if (!isdigit(dirname[i]))
168 tmp = dss_strdup(dirname);
170 ret = dss_atoi64(tmp, &num);
179 s->creation_time = num;
180 //DSS_DEBUG_LOG("%s start time: %lli\n", dirname, (long long)s->creation_time);
181 s->interval = (long long) ((now - s->creation_time)
182 / conf.unit_interval_arg / 24 / 3600);
183 if (!strcmp(dash + 1, "incomplete")) {
184 s->completion_time = -1;
185 s->flags = 0; /* neither complete, nor being deleted */
188 if (!strcmp(dash + 1, "incomplete.being_deleted")) {
189 s->completion_time = -1;
190 s->flags = SS_BEING_DELETED; /* mot cpmplete, being deleted */
194 dot = strchr(tmp, '.');
195 if (!dot || !dot[1] || dot == tmp)
197 for (i = 0; tmp[i] != '.'; i++)
198 if (!isdigit(tmp[i]))
200 tmp = dss_strdup(dash + 1);
202 ret = dss_atoi64(tmp, &num);
210 s->completion_time = num;
211 s->flags = SS_COMPLETE;
212 if (!strcmp(dot + 1, "being_deleted"))
213 s->flags |= SS_BEING_DELETED;
215 s->name = dss_strdup(dirname);
219 int64_t get_current_time(void)
223 DSS_DEBUG_LOG("now: %lli\n", (long long) now);
227 char *incomplete_name(int64_t start)
229 return make_message("%lli-incomplete", (long long)start);
232 char *being_deleted_name(struct snapshot *s)
234 if (s->flags & SS_COMPLETE)
235 return make_message("%lli-%lli.being_deleted",
236 (long long)s->creation_time,
237 (long long)s->completion_time);
238 return make_message("%lli-incomplete.being_deleted",
239 (long long)s->creation_time);
242 int complete_name(int64_t start, int64_t end, char **result)
244 struct tm start_tm, end_tm;
245 time_t *start_seconds = (time_t *) (uint64_t *)&start; /* STFU, gcc */
246 time_t *end_seconds = (time_t *) (uint64_t *)&end; /* STFU, gcc */
247 char start_str[200], end_str[200];
249 if (!localtime_r(start_seconds, &start_tm)) {
250 make_err_msg("%lli", (long long)start);
253 if (!localtime_r(end_seconds, &end_tm)) {
254 make_err_msg("%lli", (long long)end);
257 if (!strftime(start_str, sizeof(start_str), "%a_%b_%d_%Y_%H_%M_%S", &start_tm)) {
258 make_err_msg("%lli", (long long)start);
261 if (!strftime(end_str, sizeof(end_str), "%a_%b_%d_%Y_%H_%M_%S", &end_tm)) {
262 make_err_msg("%lli", (long long)end);
265 *result = make_message("%lli-%lli.%s-%s", (long long) start, (long long) end,
270 struct snapshot_list {
272 unsigned num_snapshots;
274 struct snapshot **snapshots;
276 * Array of size num_intervals + 1
278 * It contains the number of snapshots in each interval. interval_count[num_intervals]
279 * is the number of snapshots which belong to any interval greater than num_intervals.
281 unsigned *interval_count;
284 #define FOR_EACH_SNAPSHOT(s, i, sl) \
285 for ((i) = 0; (i) < (sl)->num_snapshots && ((s) = (sl)->snapshots[(i)]); (i)++)
289 #define NUM_COMPARE(x, y) ((int)((x) < (y)) - (int)((x) > (y)))
291 static int compare_snapshots(const void *a, const void *b)
293 struct snapshot *s1 = *(struct snapshot **)a;
294 struct snapshot *s2 = *(struct snapshot **)b;
295 return NUM_COMPARE(s2->creation_time, s1->creation_time);
298 /** Compute the minimum of \a a and \a b. */
299 #define DSS_MIN(a,b) ((a) < (b) ? (a) : (b))
301 int add_snapshot(const char *dirname, void *private)
303 struct snapshot_list *sl = private;
305 int ret = is_snapshot(dirname, sl->now, &s);
309 if (sl->num_snapshots >= sl->array_size) {
310 sl->array_size = 2 * sl->array_size + 1;
311 sl->snapshots = dss_realloc(sl->snapshots,
312 sl->array_size * sizeof(struct snapshot *));
314 sl->snapshots[sl->num_snapshots] = dss_malloc(sizeof(struct snapshot));
315 *(sl->snapshots[sl->num_snapshots]) = s;
316 sl->interval_count[DSS_MIN(s.interval, conf.num_intervals_arg)]++;
321 void get_snapshot_list(struct snapshot_list *sl)
323 sl->now = get_current_time();
324 sl->num_snapshots = 0;
326 sl->snapshots = NULL;
327 sl->interval_count = dss_calloc((conf.num_intervals_arg + 1) * sizeof(unsigned));
328 for_each_subdir(add_snapshot, sl);
329 qsort(sl->snapshots, sl->num_snapshots, sizeof(struct snapshot *),
333 void free_snapshot_list(struct snapshot_list *sl)
338 FOR_EACH_SNAPSHOT(s, i, sl) {
342 free(sl->interval_count);
343 sl->interval_count = NULL;
345 sl->snapshots = NULL;
346 sl->num_snapshots = 0;
349 void stop_rsync_process(void)
351 if (!rsync_pid || rsync_stopped)
353 kill(SIGSTOP, rsync_pid);
357 void restart_rsync_process(void)
359 if (!rsync_pid || !rsync_stopped)
361 kill (SIGCONT, rsync_pid);
366 * Print a log message about the exit status of a child.
368 void log_termination_msg(pid_t pid, int status)
370 if (WIFEXITED(status))
371 DSS_INFO_LOG("child %i exited. Exit status: %i\n", (int)pid,
372 WEXITSTATUS(status));
373 else if (WIFSIGNALED(status))
374 DSS_NOTICE_LOG("child %i was killed by signal %i\n", (int)pid,
377 DSS_WARNING_LOG("child %i terminated abormally\n", (int)pid);
380 int wait_for_process(pid_t pid, int *status)
384 DSS_DEBUG_LOG("Waiting for process %d to terminate\n", (int)pid);
392 if (ret == SIGCHLD) {
393 ret = waitpid(pid, status, 0);
396 if (errno != EINTR) { /* error */
397 ret = -ERRNO_TO_DSS_ERROR(errno);
401 /* SIGINT or SIGTERM */
402 DSS_WARNING_LOG("sending SIGTERM to pid %d\n", (int)pid);
406 make_err_msg("failed to wait for process %d", (int)pid);
408 log_termination_msg(pid, *status);
412 int remove_snapshot(struct snapshot *s)
414 int fds[3] = {0, 0, 0};
416 char *new_name = being_deleted_name(s);
417 int ret = dss_rename(s->name, new_name);
418 char *argv[] = {"rm", "-rf", new_name, NULL};
422 DSS_NOTICE_LOG("removing %s (interval = %i)\n", s->name, s->interval);
423 stop_rsync_process();
424 ret = dss_exec(&rm_pid, argv[0], argv, fds);
431 * return: 0: no redundant snapshots, 1: rm process started, negative: error
433 int remove_redundant_snapshot(struct snapshot_list *sl)
435 int ret, i, interval;
437 unsigned missing = 0;
439 DSS_INFO_LOG("looking for intervals containing too many snapshots\n");
440 for (interval = conf.num_intervals_arg - 1; interval >= 0; interval--) {
441 unsigned keep = num_snapshots(interval);
442 unsigned num = sl->interval_count[interval];
443 struct snapshot *victim = NULL, *prev = NULL;
444 int64_t score = LONG_MAX;
447 missing += keep - num;
448 DSS_DEBUG_LOG("interval %i: keep: %u, have: %u, missing: %u\n",
449 interval, keep, num, missing);
450 if (keep + missing >= num)
452 /* redundant snapshot in this interval, pick snapshot with lowest score */
453 FOR_EACH_SNAPSHOT(s, i, sl) {
456 //DSS_DEBUG_LOG("checking %s\n", s->name);
457 if (s->interval > interval) {
461 if (s->interval < interval)
469 /* check if s is a better victim */
470 this_score = s->creation_time - prev->creation_time;
471 assert(this_score >= 0);
472 //DSS_DEBUG_LOG("%s: score %lli\n", s->name, (long long)score);
473 if (this_score < score) {
480 if (conf.dry_run_given) {
481 dss_msg("%s would be removed (interval = %i)\n",
482 victim->name, victim->interval);
485 ret = remove_snapshot(victim);
486 return ret < 0? ret : 1;
491 int remove_outdated_snapshot(struct snapshot_list *sl)
496 DSS_INFO_LOG("looking for snapshots belonging to intervals greater than %d\n",
497 conf.num_intervals_arg);
498 FOR_EACH_SNAPSHOT(s, i, sl) {
499 if (s->interval <= conf.num_intervals_arg)
501 if (conf.dry_run_given) {
502 dss_msg("%s would be removed (interval = %i)\n",
503 s->name, s->interval);
506 ret = remove_snapshot(s);
514 int handle_rm_exit(int status)
518 if (!WIFEXITED(status)) {
519 make_err_msg("rm process %d died involuntary", (int)rm_pid);
520 ret = -E_INVOLUNTARY_EXIT;
523 es = WEXITSTATUS(status);
525 make_err_msg("rm process %d returned %d", (int)rm_pid, es);
526 ret = -E_BAD_EXIT_CODE;
535 int wait_for_rm_process(void)
537 int status, ret = wait_for_process(rm_pid, &status);
541 return handle_rm_exit(status);
544 void kill_process(pid_t pid)
548 DSS_WARNING_LOG("sending SIGTERM to pid %d\n", (int)pid);
552 int check_config(void)
554 if (conf.unit_interval_arg <= 0) {
555 make_err_msg("bad unit interval: %i", conf.unit_interval_arg);
556 return -E_INVALID_NUMBER;
558 DSS_DEBUG_LOG("unit interval: %i day(s)\n", conf.unit_interval_arg);
559 if (conf.num_intervals_arg <= 0) {
560 make_err_msg("bad number of intervals %i", conf.num_intervals_arg);
561 return -E_INVALID_NUMBER;
563 DSS_DEBUG_LOG("number of intervals: %i\n", conf.num_intervals_arg);
567 /* exits on errors */
568 void parse_config_file(int override)
573 char *old_logfile_arg = NULL;
574 int old_daemon_given = 0;
576 if (conf.config_file_given)
577 config_file = dss_strdup(conf.config_file_arg);
579 char *home = get_homedir();
580 config_file = make_message("%s/.dssrc", home);
583 if (override) { /* SIGHUP */
584 if (conf.logfile_given)
585 old_logfile_arg = dss_strdup(conf.logfile_arg);
586 old_daemon_given = conf.daemon_given;
589 ret = stat(config_file, &statbuf);
590 if (ret && conf.config_file_given) {
591 ret = -ERRNO_TO_DSS_ERROR(errno);
592 make_err_msg("failed to stat config file %s", config_file);
596 struct cmdline_parser_params params = {
597 .override = override,
602 cmdline_parser_config_file(config_file, &conf, ¶ms);
604 ret = check_config();
608 /* don't change daemon mode on SIGHUP */
609 conf.daemon_given = old_daemon_given;
612 if (conf.logfile_given)
613 free(old_logfile_arg);
614 else if (conf.daemon_given) { /* re-use old logfile */
615 conf.logfile_arg = old_logfile_arg;
616 conf.logfile_given = 1;
619 if (conf.logfile_given) {
620 logfile = open_log(conf.logfile_arg);
621 log_welcome(conf.loglevel_arg);
623 ret = dss_chdir(conf.dest_dir_arg);
628 log_err_msg(EMERG, -ret);
632 void handle_sighup(void)
634 DSS_NOTICE_LOG("SIGHUP\n");
635 parse_config_file(1);
638 int rename_incomplete_snapshot(int64_t start)
640 char *old_name, *new_name;
643 ret = complete_name(start, get_current_time(), &new_name);
646 old_name = incomplete_name(start);
647 ret = dss_rename(old_name, new_name);
649 DSS_NOTICE_LOG("%s -> %s\n", old_name, new_name);
655 int handle_rsync_exit(int status)
659 if (!WIFEXITED(status)) {
660 make_err_msg("rsync process %d died involuntary", (int)rsync_pid);
661 ret = -E_INVOLUNTARY_EXIT;
664 es = WEXITSTATUS(status);
665 if (es != 0 && es != 23 && es != 24) {
666 make_err_msg("rsync process %d returned %d", (int)rsync_pid, es);
667 ret = -E_BAD_EXIT_CODE;
670 ret = rename_incomplete_snapshot(current_snapshot_creation_time);
673 current_snapshot_creation_time = 0;
678 int get_newest_complete(const char *dirname, void *private)
680 struct edge_snapshot_data *esd = private;
682 int ret = is_snapshot(dirname, esd->now, &s);
686 if (s.flags != SS_COMPLETE) /* incomplete or being deleted */
688 if (s.creation_time < esd->snap.creation_time)
690 free(esd->snap.name);
695 __malloc char *name_of_newest_complete_snapshot(void)
697 struct edge_snapshot_data esd = {
698 .now = get_current_time(),
699 .snap = {.creation_time = -1}
701 for_each_subdir(get_newest_complete, &esd);
702 return esd.snap.name;
705 void create_rsync_argv(char ***argv, int64_t *num)
707 char *logname, *newest = name_of_newest_complete_snapshot();
710 *argv = dss_malloc((15 + conf.rsync_option_given) * sizeof(char *));
711 (*argv)[i++] = dss_strdup("rsync");
712 (*argv)[i++] = dss_strdup("-aq");
713 (*argv)[i++] = dss_strdup("--delete");
714 for (j = 0; j < conf.rsync_option_given; j++)
715 (*argv)[i++] = dss_strdup(conf.rsync_option_arg[j]);
717 DSS_INFO_LOG("using %s as reference snapshot\n", newest);
718 (*argv)[i++] = make_message("--link-dest=../%s", newest);
721 DSS_INFO_LOG("no previous snapshot found\n");
722 if (conf.exclude_patterns_given) {
723 (*argv)[i++] = dss_strdup("--exclude-from");
724 (*argv)[i++] = dss_strdup(conf.exclude_patterns_arg);
727 logname = dss_logname();
728 if (conf.remote_user_given && !strcmp(conf.remote_user_arg, logname))
729 (*argv)[i++] = dss_strdup(conf.source_dir_arg);
731 (*argv)[i++] = make_message("%s@%s:%s/", conf.remote_user_given?
732 conf.remote_user_arg : logname,
733 conf.remote_host_arg, conf.source_dir_arg);
735 *num = get_current_time();
736 (*argv)[i++] = incomplete_name(*num);
738 for (j = 0; j < i; j++)
739 DSS_DEBUG_LOG("argv[%d] = %s\n", j, (*argv)[j]);
742 void free_rsync_argv(char **argv)
745 for (i = 0; argv[i]; i++)
750 int create_snapshot(char **argv)
752 int fds[3] = {0, 0, 0};
753 char *name = incomplete_name(current_snapshot_creation_time);
755 DSS_NOTICE_LOG("creating new snapshot %s\n", name);
757 return dss_exec(&rsync_pid, argv[0], argv, fds);
760 void compute_next_snapshot_time(struct snapshot_list *sl)
762 struct timeval now, unit_interval = {.tv_sec = 24 * 3600 * conf.unit_interval_arg},
765 unsigned wanted = num_snapshots(0), num_complete_snapshots = 0;
769 gettimeofday(&now, NULL);
770 FOR_EACH_SNAPSHOT(s, i, sl) {
771 if (!(s->flags & SS_COMPLETE))
773 num_complete_snapshots++;
774 x += s->completion_time - s->creation_time;
777 if (num_complete_snapshots)
778 x /= num_complete_snapshots; /* avg time to create one snapshot */
779 x *= wanted; /* time to create all snapshots in interval 0 */
782 ret = tv_diff(&unit_interval, &tmp, &diff); /* time between creation */
784 next_snapshot_time = now;
787 tv_divide(wanted, &diff, &tmp);
788 tv_add(&now, &tmp, &next_snapshot_time);
791 void handle_signal(void)
793 int sig, ret = next_signal();
803 restart_rsync_process();
804 kill_process(rsync_pid);
805 kill_process(rm_pid);
812 ret = reap_child(&pid, &status);
815 assert(pid == rsync_pid || pid == rm_pid);
816 if (pid == rsync_pid)
817 ret = handle_rsync_exit(status);
819 ret = handle_rm_exit(status);
823 log_err_msg(ERROR, -ret);
826 int get_oldest(const char *dirname, void *private)
828 struct edge_snapshot_data *esd = private;
830 int ret = is_snapshot(dirname, esd->now, &s);
834 if (s.creation_time > esd->snap.creation_time)
836 free(esd->snap.name);
841 int remove_oldest_snapshot()
844 struct edge_snapshot_data esd = {
845 .now = get_current_time(),
846 .snap = {.creation_time = LLONG_MAX}
848 for_each_subdir(get_oldest, &esd);
849 if (!esd.snap.name) /* no snapshot found */
851 DSS_INFO_LOG("oldest snapshot: %s\n", esd.snap.name);
853 if (esd.snap.creation_time == current_snapshot_creation_time)
854 goto out; /* do not remove the snapshot currently being created */
855 ret = remove_snapshot(&esd.snap);
861 /* TODO: Also consider number of inodes. */
862 int disk_space_low(void)
864 struct disk_space ds;
865 int ret = get_disk_space(".", &ds);
869 if (conf.min_free_mb_arg)
870 if (ds.free_mb < conf.min_free_mb_arg)
872 if (conf.min_free_percent_arg)
873 if (ds.percent_free < conf.min_free_percent_arg)
878 int try_to_free_disk_space(int low_disk_space, struct snapshot_list *sl)
882 ret = remove_outdated_snapshot(sl);
883 if (ret) /* error, or we are removing something */
885 /* no outdated snapshot */
886 ret = remove_redundant_snapshot(sl);
891 DSS_WARNING_LOG("disk space low and nothing obvious to remove\n");
892 ret = remove_oldest_snapshot();
895 make_err_msg("uhuhu: not enough disk space for a single snapshot");
899 int select_loop(void)
902 struct timeval tv = {.tv_sec = 0, .tv_usec = 0};
903 struct snapshot_list sl = {.num_snapshots = 0};
906 struct timeval now, *tvp = &tv;
911 free_snapshot_list(&sl);
912 get_snapshot_list(&sl);
913 compute_next_snapshot_time(&sl);
915 FD_SET(signal_pipe, &rfds);
920 ret = dss_select(signal_pipe + 1, &rfds, NULL, tvp);
923 if (FD_ISSET(signal_pipe, &rfds)) {
929 ret = disk_space_low();
932 low_disk_space = ret;
934 stop_rsync_process();
935 ret = try_to_free_disk_space(low_disk_space, &sl);
941 restart_rsync_process();
944 /* neither rsync nor rm are running. Start rsync? */
945 gettimeofday(&now, NULL);
946 if (tv_diff(&next_snapshot_time, &now, &tv) > 0)
948 create_rsync_argv(&rsync_argv, ¤t_snapshot_creation_time);
949 ret = create_snapshot(rsync_argv);
950 free_rsync_argv(rsync_argv);
954 free_snapshot_list(&sl);
962 if (conf.dry_run_given) {
963 make_err_msg("dry_run not supported by this command");
966 ret = install_sighandler(SIGHUP);
969 return select_loop();
972 void log_disk_space(struct disk_space *ds)
974 DSS_INFO_LOG("free: %uM/%uM (%u%%), %u%% inodes unused\n",
975 ds->free_mb, ds->total_mb, ds->percent_free,
976 ds->percent_free_inodes);
982 struct snapshot_list sl;
983 struct disk_space ds;
985 ret = get_disk_space(".", &ds);
990 get_snapshot_list(&sl);
991 ret = remove_outdated_snapshot(&sl);
992 free_snapshot_list(&sl);
997 ret = wait_for_rm_process();
1002 get_snapshot_list(&sl);
1003 ret = remove_redundant_snapshot(&sl);
1004 free_snapshot_list(&sl);
1009 ret = wait_for_rm_process();
1018 int com_create(void)
1023 create_rsync_argv(&rsync_argv, ¤t_snapshot_creation_time);
1024 if (conf.dry_run_given) {
1027 for (i = 0; rsync_argv[i]; i++) {
1029 msg = make_message("%s%s%s", tmp? tmp : "",
1030 tmp? " " : "", rsync_argv[i]);
1033 dss_msg("%s\n", msg);
1037 ret = create_snapshot(rsync_argv);
1040 ret = wait_for_process(rsync_pid, &status);
1043 ret = handle_rsync_exit(status);
1045 free_rsync_argv(rsync_argv);
1052 struct snapshot_list sl;
1054 get_snapshot_list(&sl);
1055 FOR_EACH_SNAPSHOT(s, i, &sl)
1056 dss_msg("%u\t%s\n", s->interval, s->name);
1057 free_snapshot_list(&sl);
1061 __noreturn void clean_exit(int status)
1063 free(dss_error_txt);
1066 static void setup_signal_handling(void)
1070 DSS_INFO_LOG("setting up signal handlers\n");
1071 signal_pipe = signal_init(); /* always successful */
1072 ret = install_sighandler(SIGINT);
1075 ret = install_sighandler(SIGTERM);
1078 ret = install_sighandler(SIGCHLD);
1083 DSS_EMERG_LOG("could not install signal handlers\n");
1087 int main(int argc, char **argv)
1090 struct cmdline_parser_params params = {
1093 .check_required = 0,
1094 .check_ambiguity = 0
1097 cmdline_parser_ext(argc, argv, &conf, ¶ms); /* aborts on errors */
1098 parse_config_file(0);
1099 if (conf.daemon_given)
1101 setup_signal_handling();
1102 ret = call_command_handler();
1104 log_err_msg(EMERG, -ret);
1105 clean_exit(ret >= 0? EXIT_SUCCESS : EXIT_FAILURE);