/* * Amanda, The Advanced Maryland Automatic Network Disk Archiver * Copyright (c) 1991-1998 University of Maryland at College Park * All Rights Reserved. * * Permission to use, copy, modify, distribute, and sell this software and its * documentation for any purpose is hereby granted without fee, provided that * the above copyright notice appear in all copies and that both that * copyright notice and this permission notice appear in supporting * documentation, and that the name of U.M. not be used in advertising or * publicity pertaining to distribution of the software without specific, * written prior permission. U.M. makes no representations about the * suitability of this software for any purpose. It is provided "as is" * without express or implied warranty. * * U.M. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL U.M. * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. * * Authors: the Amanda Development Team. Its members are listed in a * file named AUTHORS, in the root directory of this distribution. */ /* * $Id: driver.c,v 1.198.2.6 2006/12/27 14:44:48 martinea Exp $ * * controlling process for the Amanda backup system */ /* * XXX possibly modify tape queue to be cognizant of how much room is left on * tape. Probably not effective though, should do this in planner. */ #define HOLD_DEBUG #include "amanda.h" #include "clock.h" #include "conffile.h" #include "diskfile.h" #include "event.h" #include "holding.h" #include "infofile.h" #include "logfile.h" #include "statfs.h" #include "version.h" #include "driverio.h" #include "server_util.h" static disklist_t waitq, runq, tapeq, roomq; static int pending_aborts; static disk_t *taper_disk; static int degraded_mode; static off_t reserved_space; static off_t total_disksize; static char *dumper_program; static char *chunker_program; static int inparallel; static int nodump = 0; static off_t tape_length = (off_t)0; static off_t tape_left = (off_t)0; static int current_tape = 1; static int conf_taperalgo; static int conf_runtapes; static time_t sleep_time; static int idle_reason; static char *driver_timestamp; static char *hd_driver_timestamp; static am_host_t *flushhost = NULL; static int need_degraded=0; static event_handle_t *dumpers_ev_time = NULL; static event_handle_t *schedule_ev_read = NULL; static int wait_children(int count); static void wait_for_children(void); static void allocate_bandwidth(interface_t *ip, unsigned long kps); static int assign_holdingdisk(assignedhd_t **holdp, disk_t *diskp); static void adjust_diskspace(disk_t *diskp, cmd_t cmd); static void delete_diskspace(disk_t *diskp); static assignedhd_t **build_diskspace(char *destname); static int client_constrained(disk_t *dp); static void deallocate_bandwidth(interface_t *ip, unsigned long kps); static void dump_schedule(disklist_t *qp, char *str); static int dump_to_tape(disk_t *dp); static assignedhd_t **find_diskspace(off_t size, int *cur_idle, assignedhd_t *preferred); static unsigned long free_kps(interface_t *ip); static off_t free_space(void); static void dumper_result(disk_t *dp); static void handle_dumper_result(void *); static void handle_chunker_result(void *); static void handle_dumpers_time(void *); static void handle_taper_result(void *); static void holdingdisk_state(char *time_str); static dumper_t *idle_dumper(void); static void interface_state(char *time_str); static int queue_length(disklist_t q); static disklist_t read_flush(void); static void read_schedule(void *cookie); static void short_dump_state(void); static void startaflush(void); static void start_degraded_mode(disklist_t *queuep); static void start_some_dumps(disklist_t *rq); static void continue_port_dumps(void); static void update_failed_dump_to_tape(disk_t *); #if 0 static void dump_state(const char *str); #endif int main(int main_argc, char **main_argv); static const char *idle_strings[] = { #define NOT_IDLE 0 "not-idle", #define IDLE_NO_DUMPERS 1 "no-dumpers", #define IDLE_START_WAIT 2 "start-wait", #define IDLE_NO_HOLD 3 "no-hold", #define IDLE_CLIENT_CONSTRAINED 4 "client-constrained", #define IDLE_NO_DISKSPACE 5 "no-diskspace", #define IDLE_TOO_LARGE 6 "file-too-large", #define IDLE_NO_BANDWIDTH 7 "no-bandwidth", #define IDLE_TAPER_WAIT 8 "taper-wait", }; int main( int main_argc, char ** main_argv) { disklist_t origq; disk_t *diskp; int dsk; dumper_t *dumper; char *newdir = NULL; generic_fs_stats_t fs; holdingdisk_t *hdp; unsigned long malloc_hist_1, malloc_size_1; unsigned long malloc_hist_2, malloc_size_2; unsigned long reserve = 100; char *conffile; char *conf_diskfile; cmd_t cmd; int result_argc; char *result_argv[MAX_ARGS+1]; char *taper_program; char *conf_tapetype; tapetype_t *tape; char *line; int new_argc, my_argc; char **new_argv, **my_argv; safe_fd(-1, 0); setvbuf(stdout, (char *)NULL, (int)_IOLBF, 0); setvbuf(stderr, (char *)NULL, (int)_IOLBF, 0); set_pname("driver"); dbopen(DBG_SUBDIR_SERVER); atexit(wait_for_children); /* Don't die when child closes pipe */ signal(SIGPIPE, SIG_IGN); malloc_size_1 = malloc_inuse(&malloc_hist_1); erroutput_type = (ERR_AMANDALOG|ERR_INTERACTIVE); set_logerror(logerror); startclock(); parse_server_conf(main_argc, main_argv, &new_argc, &new_argv); my_argc = new_argc; my_argv = new_argv; printf("%s: pid %ld executable %s version %s\n", get_pname(), (long) getpid(), my_argv[0], version()); if (my_argc > 1) { config_name = stralloc(my_argv[1]); config_dir = vstralloc(CONFIG_DIR, "/", config_name, "/", NULL); if(my_argc > 2) { if(strncmp(my_argv[2], "nodump", 6) == 0) { nodump = 1; } } } else { char my_cwd[STR_SIZE]; if (getcwd(my_cwd, SIZEOF(my_cwd)) == NULL) { error("cannot determine current working directory"); /*NOTREACHED*/ } config_dir = stralloc2(my_cwd, "/"); if ((config_name = strrchr(my_cwd, '/')) != NULL) { config_name = stralloc(config_name + 1); } } safe_cd(); conffile = stralloc2(config_dir, CONFFILE_NAME); if(read_conffile(conffile)) { error("errors processing config file \"%s\"", conffile); /*NOTREACHED*/ } amfree(conffile); dbrename(config_name, DBG_SUBDIR_SERVER); report_bad_conf_arg(); amfree(driver_timestamp); /* read timestamp from stdin */ while ((line = agets(stdin)) != NULL) { if (line[0] != '\0') break; amfree(line); } if ( line == NULL ) { error("Did not get DATE line from planner"); /*NOTREACHED*/ } driver_timestamp = alloc(15); strncpy(driver_timestamp, &line[5], 14); driver_timestamp[14] = '\0'; amfree(line); log_add(L_START,"date %s", driver_timestamp); /* check that we don't do many dump in a day and usetimestamps is off */ if(strlen(driver_timestamp) == 8) { if (!nodump) { char *conf_logdir = getconf_str(CNF_LOGDIR); char *logfile = vstralloc(conf_logdir, "/log.", driver_timestamp, ".0", NULL); char *oldlogfile = vstralloc(conf_logdir, "/oldlog/log.", driver_timestamp, ".0", NULL); if(access(logfile, F_OK) == 0 || access(oldlogfile, F_OK) == 0) { log_add(L_WARNING, "WARNING: This is not the first amdump run today. Enable the usetimestamps option in the configuration file if you want to run amdump more than once per calendar day."); } amfree(oldlogfile); amfree(logfile); } hd_driver_timestamp = construct_timestamp(NULL); } else { hd_driver_timestamp = stralloc(driver_timestamp); } taper_program = vstralloc(libexecdir, "/", "taper", versionsuffix(), NULL); dumper_program = vstralloc(libexecdir, "/", "dumper", versionsuffix(), NULL); chunker_program = vstralloc(libexecdir, "/", "chunker", versionsuffix(), NULL); conf_taperalgo = getconf_taperalgo(CNF_TAPERALGO); conf_tapetype = getconf_str(CNF_TAPETYPE); conf_runtapes = getconf_int(CNF_RUNTAPES); tape = lookup_tapetype(conf_tapetype); tape_length = tapetype_get_length(tape); printf("driver: tape size " OFF_T_FMT "\n", (OFF_T_FMT_TYPE)tape_length); /* start initializing: read in databases */ conf_diskfile = getconf_str(CNF_DISKFILE); if (*conf_diskfile == '/') { conf_diskfile = stralloc(conf_diskfile); } else { conf_diskfile = stralloc2(config_dir, conf_diskfile); } if (read_diskfile(conf_diskfile, &origq) < 0) { error("could not load disklist \"%s\"", conf_diskfile); /*NOTREACHED*/ } amfree(conf_diskfile); /* set up any configuration-dependent variables */ inparallel = getconf_int(CNF_INPARALLEL); reserve = (unsigned long)getconf_int(CNF_RESERVE); total_disksize = (off_t)0; for(hdp = getconf_holdingdisks(), dsk = 0; hdp != NULL; hdp = hdp->next, dsk++) { hdp->up = (void *)alloc(SIZEOF(holdalloc_t)); holdalloc(hdp)->allocated_dumpers = 0; holdalloc(hdp)->allocated_space = (off_t)0; if(get_fs_stats(holdingdisk_get_diskdir(hdp), &fs) == -1 || access(holdingdisk_get_diskdir(hdp), W_OK) == -1) { log_add(L_WARNING, "WARNING: ignoring holding disk %s: %s\n", holdingdisk_get_diskdir(hdp), strerror(errno)); hdp->disksize = 0L; continue; } if(fs.avail != (off_t)-1) { if(hdp->disksize > (off_t)0) { if(hdp->disksize > fs.avail) { log_add(L_WARNING, "WARNING: %s: " OFF_T_FMT " KB requested, " "but only " OFF_T_FMT " KB available.", holdingdisk_get_diskdir(hdp), (OFF_T_FMT_TYPE)hdp->disksize, (OFF_T_FMT_TYPE)fs.avail); hdp->disksize = fs.avail; } } else if((fs.avail + hdp->disksize) < (off_t)0) { log_add(L_WARNING, "WARNING: %s: not " OFF_T_FMT " KB free.", holdingdisk_get_diskdir(hdp), -hdp->disksize); hdp->disksize = (off_t)0; continue; } else hdp->disksize += fs.avail; } printf("driver: adding holding disk %d dir %s size " OFF_T_FMT " chunksize " OFF_T_FMT "\n", dsk, holdingdisk_get_diskdir(hdp), (OFF_T_FMT_TYPE)hdp->disksize, (OFF_T_FMT_TYPE)(holdingdisk_get_chunksize(hdp))); newdir = newvstralloc(newdir, holdingdisk_get_diskdir(hdp), "/", hd_driver_timestamp, NULL); if(!mkholdingdir(newdir)) { hdp->disksize = (off_t)0; } total_disksize += hdp->disksize; } reserved_space = total_disksize * (off_t)(reserve / 100); printf("reserving " OFF_T_FMT " out of " OFF_T_FMT " for degraded-mode dumps\n", (OFF_T_FMT_TYPE)reserved_space, (OFF_T_FMT_TYPE)free_space()); amfree(newdir); if(inparallel > MAX_DUMPERS) inparallel = MAX_DUMPERS; /* taper takes a while to get going, so start it up right away */ init_driverio(); if(conf_runtapes > 0) { startup_tape_process(taper_program); taper_cmd(START_TAPER, driver_timestamp, NULL, 0, NULL); } /* fire up the dumpers now while we are waiting */ if(!nodump) startup_dump_processes(dumper_program, inparallel, driver_timestamp); /* * Read schedule from stdin. Usually, this is a pipe from planner, * so the effect is that we wait here for the planner to * finish, but meanwhile the taper is rewinding the tape, reading * the label, checking it, writing a new label and all that jazz * in parallel with the planner. */ runq.head = NULL; runq.tail = NULL; waitq = origq; tapeq = read_flush(); roomq.head = roomq.tail = NULL; log_add(L_STATS, "startup time %s", walltime_str(curclock())); printf("driver: start time %s inparallel %d bandwidth %lu diskspace " OFF_T_FMT " ", walltime_str(curclock()), inparallel, free_kps((interface_t *)0), (OFF_T_FMT_TYPE)free_space()); printf(" dir %s datestamp %s driver: drain-ends tapeq %s big-dumpers %s\n", "OBSOLETE", driver_timestamp, taperalgo2str(conf_taperalgo), getconf_str(CNF_DUMPORDER)); fflush(stdout); /* ok, planner is done, now lets see if the tape is ready */ if(conf_runtapes > 0) { cmd = getresult(taper, 1, &result_argc, result_argv, MAX_ARGS+1); if(cmd != TAPER_OK) { /* no tape, go into degraded mode: dump to holding disk */ need_degraded=1; } } else { need_degraded=1; } tape_left = tape_length; taper_busy = 0; taper_disk = NULL; taper_ev_read = NULL; if(!need_degraded) startaflush(); if(!nodump) schedule_ev_read = event_register((event_id_t)0, EV_READFD, read_schedule, NULL); short_dump_state(); event_loop(0); /* handle any remaining dumps by dumping directly to tape, if possible */ while(!empty(runq) && taper > 0) { diskp = dequeue_disk(&runq); if (diskp->to_holdingdisk == HOLD_REQUIRED) { log_add(L_FAIL, "%s %s %s %d [%s]", diskp->host->hostname, diskp->name, sched(diskp)->datestamp, sched(diskp)->level, "can't dump required holdingdisk"); } else if (!degraded_mode) { int rc = dump_to_tape(diskp); if(rc == 1) log_add(L_INFO, "%s %s %d [dump to tape failed, will try again]", diskp->host->hostname, diskp->name, sched(diskp)->level); else if(rc == 2) log_add(L_FAIL, "%s %s %s %d [dump to tape failed]", diskp->host->hostname, diskp->name, sched(diskp)->datestamp, sched(diskp)->level); } else log_add(L_FAIL, "%s %s %s %d [%s]", diskp->host->hostname, diskp->name, sched(diskp)->datestamp, sched(diskp)->level, diskp->to_holdingdisk == HOLD_AUTO ? "no more holding disk space" : "can't dump no-hold disk in degraded mode"); } short_dump_state(); /* for amstatus */ printf("driver: QUITTING time %s telling children to quit\n", walltime_str(curclock())); fflush(stdout); if(!nodump) { for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) { if(dumper->fd >= 0) dumper_cmd(dumper, QUIT, NULL); } } if(taper >= 0) { taper_cmd(QUIT, NULL, NULL, 0, NULL); } /* wait for all to die */ wait_children(600); for(hdp = getconf_holdingdisks(); hdp != NULL; hdp = hdp->next) { cleanup_holdingdisk(holdingdisk_get_diskdir(hdp), 0); amfree(hdp->up); } amfree(newdir); check_unfree_serial(); printf("driver: FINISHED time %s\n", walltime_str(curclock())); fflush(stdout); log_add(L_FINISH,"date %s time %s", driver_timestamp, walltime_str(curclock())); amfree(driver_timestamp); free_new_argv(new_argc, new_argv); amfree(dumper_program); amfree(taper_program); amfree(config_dir); amfree(config_name); malloc_size_2 = malloc_inuse(&malloc_hist_2); if(malloc_size_1 != malloc_size_2) { malloc_list(fileno(stderr), malloc_hist_1, malloc_hist_2); } dbclose(); return 0; } /* sleep up to count seconds, and wait for terminating child process */ /* if sleep is negative, this function will not timeout */ /* exit once all child process are finished or the timout expired */ /* return 0 if no more children to wait */ /* return 1 if some children are still alive */ static int wait_children(int count) { pid_t pid; amwait_t retstat; char *who; char *what; int code=0; dumper_t *dumper; int wait_errno; do { do { pid = waitpid((pid_t)-1, &retstat, WNOHANG); wait_errno = errno; if (pid > 0) { what = NULL; if (! WIFEXITED(retstat)) { what = "signal"; code = WTERMSIG(retstat); } else if (WEXITSTATUS(retstat) != 0) { what = "code"; code = WEXITSTATUS(retstat); } who = NULL; for (dumper = dmptable; dumper < dmptable + inparallel; dumper++) { if (pid == dumper->pid) { who = stralloc(dumper->name); dumper->pid = -1; break; } if (dumper->chunker && pid == dumper->chunker->pid) { who = stralloc(dumper->chunker->name); dumper->chunker->pid = -1; break; } } if (who == NULL && pid == taper_pid) { who = stralloc("taper"); taper_pid = -1; } if(what != NULL && who == NULL) { who = stralloc("unknown"); } if(who && what) { log_add(L_WARNING, "%s pid %u exited with %s %d\n", who, (unsigned)pid, what, code); printf("driver: %s pid %u exited with %s %d\n", who, (unsigned)pid, what, code); } amfree(who); } } while (pid > 0 || wait_errno == EINTR); if (errno != ECHILD) sleep(1); if (count > 0) count--; } while ((errno != ECHILD) && (count != 0)); return (errno != ECHILD); } static void kill_children(int signal) { dumper_t *dumper; if(!nodump) { for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) { if (!dumper->down && dumper->pid > 1) { printf("driver: sending signal %d to %s pid %u\n", signal, dumper->name, (unsigned)dumper->pid); if (kill(dumper->pid, signal) == -1 && errno == ESRCH) { if (dumper->chunker) dumper->chunker->pid = 0; } if (dumper->chunker && dumper->chunker->pid > 1) { printf("driver: sending signal %d to %s pid %u\n", signal, dumper->chunker->name, (unsigned)dumper->chunker->pid); if (kill(dumper->chunker->pid, signal) == -1 && errno == ESRCH) dumper->chunker->pid = 0; } } } } if(taper_pid > 1) printf("driver: sending signal %d to %s pid %u\n", signal, "taper", (unsigned)taper_pid); if (kill(taper_pid, signal) == -1 && errno == ESRCH) taper_pid = 0; } static void wait_for_children(void) { dumper_t *dumper; if(!nodump) { for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) { if (dumper->pid > 1 && dumper->fd >= 0) { dumper_cmd(dumper, QUIT, NULL); if (dumper->chunker && dumper->chunker->pid > 1 && dumper->chunker->fd >= 0) chunker_cmd(dumper->chunker, QUIT, NULL); } } } if(taper_pid > 1 && taper > 0) { taper_cmd(QUIT, NULL, NULL, 0, NULL); } if(wait_children(60) == 0) return; kill_children(SIGHUP); if(wait_children(60) == 0) return; kill_children(SIGKILL); if(wait_children(-1) == 0) return; } static void startaflush(void) { disk_t *dp = NULL; disk_t *fit = NULL; char *datestamp; int extra_tapes = 0; char *qname; if(!degraded_mode && !taper_busy && !empty(tapeq)) { datestamp = sched(tapeq.head)->datestamp; switch(conf_taperalgo) { case ALGO_FIRST: dp = dequeue_disk(&tapeq); break; case ALGO_FIRSTFIT: fit = tapeq.head; while (fit != NULL) { extra_tapes = (fit->tape_splitsize > (off_t)0) ? conf_runtapes - current_tape : 0; if(sched(fit)->act_size <= (tape_left + tape_length * (off_t)extra_tapes) && strcmp(sched(fit)->datestamp, datestamp) <= 0) { dp = fit; fit = NULL; } else { fit = fit->next; } } if(dp) remove_disk(&tapeq, dp); break; case ALGO_LARGEST: fit = dp = tapeq.head; while (fit != NULL) { if(sched(fit)->act_size > sched(dp)->act_size && strcmp(sched(fit)->datestamp, datestamp) <= 0) { dp = fit; } fit = fit->next; } if(dp) remove_disk(&tapeq, dp); break; case ALGO_LARGESTFIT: fit = tapeq.head; while (fit != NULL) { extra_tapes = (fit->tape_splitsize > (off_t)0) ? conf_runtapes - current_tape : 0; if(sched(fit)->act_size <= (tape_left + tape_length * (off_t)extra_tapes) && (!dp || sched(fit)->act_size > sched(dp)->act_size) && strcmp(sched(fit)->datestamp, datestamp) <= 0) { dp = fit; } fit = fit->next; } if(dp) remove_disk(&tapeq, dp); break; case ALGO_SMALLEST: break; case ALGO_LAST: dp = tapeq.tail; remove_disk(&tapeq, dp); break; } if(!dp) { /* ALGO_SMALLEST, or default if nothing fit. */ if(conf_taperalgo != ALGO_SMALLEST) { fprintf(stderr, "driver: startaflush: Using SMALLEST because nothing fit\n"); } fit = dp = tapeq.head; while (fit != NULL) { if(sched(fit)->act_size < sched(dp)->act_size && strcmp(sched(fit)->datestamp, datestamp) <= 0) { dp = fit; } fit = fit->next; } if(dp) remove_disk(&tapeq, dp); } if(taper_ev_read == NULL) { taper_ev_read = event_register((event_id_t)taper, EV_READFD, handle_taper_result, NULL); } if (dp) { taper_disk = dp; taper_busy = 1; qname = quote_string(dp->name); taper_cmd(FILE_WRITE, dp, sched(dp)->destname, sched(dp)->level, sched(dp)->datestamp); fprintf(stderr,"driver: startaflush: %s %s %s " OFF_T_FMT " " OFF_T_FMT "\n", taperalgo2str(conf_taperalgo), dp->host->hostname, qname, (OFF_T_FMT_TYPE)sched(taper_disk)->act_size, (OFF_T_FMT_TYPE)tape_left); if(sched(dp)->act_size <= tape_left) tape_left -= sched(dp)->act_size; else tape_left = (off_t)0; amfree(qname); } else { error("FATAL: Taper marked busy and no work found."); /*NOTREACHED*/ } } else if(!taper_busy && taper_ev_read != NULL) { event_release(taper_ev_read); taper_ev_read = NULL; } } static int client_constrained( disk_t * dp) { disk_t *dp2; /* first, check if host is too busy */ if(dp->host->inprogress >= dp->host->maxdumps) { return 1; } /* next, check conflict with other dumps on same spindle */ if(dp->spindle == -1) { /* but spindle -1 never conflicts by def. */ return 0; } for(dp2 = dp->host->disks; dp2 != NULL; dp2 = dp2->hostnext) if(dp2->inprogress && dp2->spindle == dp->spindle) { return 1; } return 0; } static void start_some_dumps( disklist_t * rq) { int cur_idle; disk_t *diskp, *delayed_diskp, *diskp_accept; assignedhd_t **holdp=NULL, **holdp_accept; const time_t now = time(NULL); cmd_t cmd; int result_argc; char *result_argv[MAX_ARGS+1]; chunker_t *chunker; dumper_t *dumper; char dumptype; char *dumporder; idle_reason = IDLE_NO_DUMPERS; sleep_time = 0; if(dumpers_ev_time != NULL) { event_release(dumpers_ev_time); dumpers_ev_time = NULL; } for (dumper = dmptable; dumper < dmptable+inparallel; dumper++) { if( dumper->busy ) { continue; } if (dumper->ev_read != NULL) { event_release(dumper->ev_read); dumper->ev_read = NULL; } /* * A potential problem with starting from the bottom of the dump time * distribution is that a slave host will have both one of the shortest * and one of the longest disks, so starting its shortest disk first will * tie up the host and eliminate its longest disk from consideration the * first pass through. This could cause a big delay in starting that long * disk, which could drag out the whole night's dumps. * * While starting from the top of the dump time distribution solves the * above problem, this turns out to be a bad idea, because the big dumps * will almost certainly pack the holding disk completely, leaving no * room for even one small dump to start. This ends up shutting out the * small-end dumpers completely (they stay idle). * * The introduction of multiple simultaneous dumps to one host alleviates * the biggest&smallest dumps problem: both can be started at the * beginning. */ diskp_accept = NULL; holdp_accept = NULL; delayed_diskp = NULL; cur_idle = NOT_IDLE; dumporder = getconf_str(CNF_DUMPORDER); if(strlen(dumporder) > (size_t)(dumper-dmptable)) { dumptype = dumporder[dumper-dmptable]; } else { if(dumper-dmptable < 3) dumptype = 't'; else dumptype = 'T'; } for(diskp = rq->head; diskp != NULL; diskp = diskp->next) { assert(diskp->host != NULL && sched(diskp) != NULL); if (diskp->host->start_t > now) { cur_idle = max(cur_idle, IDLE_START_WAIT); if (delayed_diskp == NULL || sleep_time > diskp->host->start_t) { delayed_diskp = diskp; sleep_time = diskp->host->start_t; } } else if(diskp->start_t > now) { cur_idle = max(cur_idle, IDLE_START_WAIT); if (delayed_diskp == NULL || sleep_time > diskp->start_t) { delayed_diskp = diskp; sleep_time = diskp->start_t; } } else if (diskp->host->netif->curusage > 0 && sched(diskp)->est_kps > free_kps(diskp->host->netif)) { cur_idle = max(cur_idle, IDLE_NO_BANDWIDTH); } else if(sched(diskp)->no_space) { cur_idle = max(cur_idle, IDLE_NO_DISKSPACE); } else if (diskp->to_holdingdisk == HOLD_NEVER) { cur_idle = max(cur_idle, IDLE_NO_HOLD); } else if ((holdp = find_diskspace(sched(diskp)->est_size, &cur_idle, NULL)) == NULL) { cur_idle = max(cur_idle, IDLE_NO_DISKSPACE); } else if (client_constrained(diskp)) { free_assignedhd(holdp); cur_idle = max(cur_idle, IDLE_CLIENT_CONSTRAINED); } else { /* disk fits, dump it */ int accept = !diskp_accept; if(!accept) { switch(dumptype) { case 's': accept = (sched(diskp)->est_size < sched(diskp_accept)->est_size); break; case 'S': accept = (sched(diskp)->est_size > sched(diskp_accept)->est_size); break; case 't': accept = (sched(diskp)->est_time < sched(diskp_accept)->est_time); break; case 'T': accept = (sched(diskp)->est_time > sched(diskp_accept)->est_time); break; case 'b': accept = (sched(diskp)->est_kps < sched(diskp_accept)->est_kps); break; case 'B': accept = (sched(diskp)->est_kps > sched(diskp_accept)->est_kps); break; default: log_add(L_WARNING, "Unknown dumporder character \'%c\', using 's'.\n", dumptype); accept = (sched(diskp)->est_size < sched(diskp_accept)->est_size); break; } } if(accept) { if( !diskp_accept || !degraded_mode || diskp->priority >= diskp_accept->priority) { if(holdp_accept) free_assignedhd(holdp_accept); diskp_accept = diskp; holdp_accept = holdp; } else { free_assignedhd(holdp); } } else { free_assignedhd(holdp); } } } diskp = diskp_accept; holdp = holdp_accept; idle_reason = max(idle_reason, cur_idle); /* * If we have no disk at this point, and there are disks that * are delayed, then schedule a time event to call this dumper * with the disk with the shortest delay. */ if (diskp == NULL && delayed_diskp != NULL) { assert(sleep_time > now); sleep_time -= now; dumpers_ev_time = event_register((event_id_t)sleep_time, EV_TIME, handle_dumpers_time, &runq); return; } else if (diskp != NULL) { sched(diskp)->act_size = (off_t)0; allocate_bandwidth(diskp->host->netif, sched(diskp)->est_kps); sched(diskp)->activehd = assign_holdingdisk(holdp, diskp); amfree(holdp); sched(diskp)->destname = newstralloc(sched(diskp)->destname, sched(diskp)->holdp[0]->destname); diskp->host->inprogress++; /* host is now busy */ diskp->inprogress = 1; sched(diskp)->dumper = dumper; sched(diskp)->timestamp = now; dumper->busy = 1; /* dumper is now busy */ dumper->dp = diskp; /* link disk to dumper */ remove_disk(rq, diskp); /* take it off the run queue */ sched(diskp)->origsize = (off_t)-1; sched(diskp)->dumpsize = (off_t)-1; sched(diskp)->dumptime = (time_t)0; sched(diskp)->tapetime = (time_t)0; chunker = dumper->chunker; chunker->result = LAST_TOK; dumper->result = LAST_TOK; startup_chunk_process(chunker,chunker_program); chunker_cmd(chunker, START, (void *)driver_timestamp); chunker->dumper = dumper; chunker_cmd(chunker, PORT_WRITE, diskp); cmd = getresult(chunker->fd, 1, &result_argc, result_argv, MAX_ARGS+1); if(cmd != PORT) { assignedhd_t **h=NULL; int activehd; printf("driver: did not get PORT from %s for %s:%s\n", chunker->name, diskp->host->hostname, diskp->name); fflush(stdout); deallocate_bandwidth(diskp->host->netif, sched(diskp)->est_kps); h = sched(diskp)->holdp; activehd = sched(diskp)->activehd; h[activehd]->used = 0; holdalloc(h[activehd]->disk)->allocated_dumpers--; adjust_diskspace(diskp, DONE); delete_diskspace(diskp); diskp->host->inprogress--; diskp->inprogress = 0; sched(diskp)->dumper = NULL; dumper->busy = 0; dumper->dp = NULL; sched(diskp)->attempted++; free_serial_dp(diskp); if(sched(diskp)->attempted < 2) enqueue_disk(rq, diskp); } else { dumper->ev_read = event_register((event_id_t)dumper->fd, EV_READFD, handle_dumper_result, dumper); chunker->ev_read = event_register((event_id_t)chunker->fd, EV_READFD, handle_chunker_result, chunker); dumper->output_port = atoi(result_argv[2]); dumper_cmd(dumper, PORT_DUMP, diskp); } diskp->host->start_t = now + 15; } } } /* * This gets called when a dumper is delayed for some reason. It may * be because a disk has a delayed start, or amanda is constrained * by network or disk limits. */ static void handle_dumpers_time( void * cookie) { disklist_t *runq = cookie; event_release(dumpers_ev_time); dumpers_ev_time = NULL; start_some_dumps(runq); } static void dump_schedule( disklist_t *qp, char * str) { disk_t *dp; char *qname; printf("dump of driver schedule %s:\n--------\n", str); for(dp = qp->head; dp != NULL; dp = dp->next) { qname = quote_string(dp->name); printf(" %-20s %-25s lv %d t %5lu s " OFF_T_FMT " p %d\n", dp->host->hostname, qname, sched(dp)->level, sched(dp)->est_time, (OFF_T_FMT_TYPE)sched(dp)->est_size, sched(dp)->priority); amfree(qname); } printf("--------\n"); } static void start_degraded_mode( /*@keep@*/ disklist_t *queuep) { disk_t *dp; disklist_t newq; off_t est_full_size; char *qname; if (taper_ev_read != NULL) { event_release(taper_ev_read); taper_ev_read = NULL; } newq.head = newq.tail = 0; dump_schedule(queuep, "before start degraded mode"); est_full_size = (off_t)0; while(!empty(*queuep)) { dp = dequeue_disk(queuep); qname = quote_string(dp->name); if(sched(dp)->level != 0) /* go ahead and do the disk as-is */ enqueue_disk(&newq, dp); else { if (reserved_space + est_full_size + sched(dp)->est_size <= total_disksize) { enqueue_disk(&newq, dp); est_full_size += sched(dp)->est_size; } else if(sched(dp)->degr_level != -1) { sched(dp)->level = sched(dp)->degr_level; sched(dp)->dumpdate = sched(dp)->degr_dumpdate; sched(dp)->est_nsize = sched(dp)->degr_nsize; sched(dp)->est_csize = sched(dp)->degr_csize; sched(dp)->est_time = sched(dp)->degr_time; sched(dp)->est_kps = sched(dp)->degr_kps; enqueue_disk(&newq, dp); } else { log_add(L_FAIL,"%s %s %s %d [can't switch to incremental dump]", dp->host->hostname, qname, sched(dp)->datestamp, sched(dp)->level); } } amfree(qname); } /*@i@*/ *queuep = newq; degraded_mode = 1; dump_schedule(queuep, "after start degraded mode"); } static void continue_port_dumps(void) { disk_t *dp, *ndp; assignedhd_t **h; int active_dumpers=0, busy_dumpers=0, i; dumper_t *dumper; /* First we try to grant diskspace to some dumps waiting for it. */ for( dp = roomq.head; dp; dp = ndp ) { ndp = dp->next; /* find last holdingdisk used by this dump */ for( i = 0, h = sched(dp)->holdp; h[i+1]; i++ ) { (void)h; /* Quiet lint */ } /* find more space */ h = find_diskspace( sched(dp)->est_size - sched(dp)->act_size, &active_dumpers, h[i] ); if( h ) { for(dumper = dmptable; dumper < dmptable + inparallel && dumper->dp != dp; dumper++) { (void)dp; /* Quiet lint */ } assert( dumper < dmptable + inparallel ); sched(dp)->activehd = assign_holdingdisk( h, dp ); chunker_cmd( dumper->chunker, CONTINUE, dp ); amfree(h); remove_disk( &roomq, dp ); } } /* So for some disks there is less holding diskspace available than * was asked for. Possible reasons are * a) diskspace has been allocated for other dumps which are * still running or already being written to tape * b) all other dumps have been suspended due to lack of diskspace * c) this dump doesn't fit on all the holding disks * Case a) is not a problem. We just wait for the diskspace to * be freed by moving the current disk to a queue. * If case b) occurs, we have a deadlock situation. We select * a dump from the queue to be aborted and abort it. It will * be retried later dumping to disk. * If case c) is detected, the dump is aborted. Next time * it will be dumped directly to tape. Actually, case c is a special * manifestation of case b) where only one dumper is busy. */ for(dp=NULL, dumper = dmptable; dumper < (dmptable+inparallel); dumper++) { if( dumper->busy ) { busy_dumpers++; if( !find_disk(&roomq, dumper->dp) ) { active_dumpers++; } else if( !dp || sched(dp)->est_size > sched(dumper->dp)->est_size ) { dp = dumper->dp; } } } if((dp != NULL) && (active_dumpers == 0) && (busy_dumpers > 0) && ((!taper_busy && empty(tapeq)) || degraded_mode) && pending_aborts == 0 ) { /* not case a */ if( busy_dumpers == 1 ) { /* case c */ sched(dp)->no_space = 1; } /* case b */ /* At this time, dp points to the dump with the smallest est_size. * We abort that dump, hopefully not wasting too much time retrying it. */ remove_disk( &roomq, dp ); chunker_cmd( sched(dp)->dumper->chunker, ABORT, NULL); dumper_cmd( sched(dp)->dumper, ABORT, NULL ); pending_aborts++; } } static void handle_taper_result( void * cookie) { disk_t *dp; off_t filenum; cmd_t cmd; int result_argc; char *result_argv[MAX_ARGS+1]; int avail_tapes = 0; (void)cookie; /* Quiet unused parameter warning */ assert(cookie == NULL); do { short_dump_state(); cmd = getresult(taper, 1, &result_argc, result_argv, MAX_ARGS+1); switch(cmd) { case PARTIAL: case DONE: /* DONE