2 * Copyright 2006-2007, Sine Nomine Associates and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
12 * online salvager daemon
15 /* Main program file. Define globals. */
18 #include <afsconfig.h>
19 #include <afs/param.h>
23 #ifdef HAVE_SYS_FILE_H
28 #include <WINNT/afsevent.h>
32 #define WCOREDUMP(x) ((x) & 0200)
37 #include <afs/afsint.h>
38 #include <rx/rx_queue.h>
40 #if !defined(AFS_SGI_ENV) && !defined(AFS_NT40_ENV)
41 #if defined(AFS_VFSINCL_ENV)
42 #include <sys/vnode.h>
44 #include <sys/fs/ufs_inode.h>
46 #if defined(AFS_DARWIN_ENV) || defined(AFS_XBSD_ENV)
47 #include <ufs/ufs/dinode.h>
48 #include <ufs/ffs/fs.h>
50 #include <ufs/inode.h>
53 #else /* AFS_VFSINCL_ENV */
54 #if !defined(AFS_LINUX20_ENV) && !defined(AFS_XBSD_ENV) && !defined(AFS_DARWIN_ENV)
55 #include <sys/inode.h>
57 #endif /* AFS_VFSINCL_ENV */
58 #endif /* AFS_SGI_ENV */
61 #include <sys/lockf.h>
64 #include <checklist.h>
66 #if defined(AFS_SGI_ENV)
69 #if defined(AFS_SUN_ENV) || defined(AFS_SUN5_ENV)
71 #include <sys/mnttab.h>
72 #include <sys/mntent.h>
77 #endif /* AFS_SGI_ENV */
78 #endif /* AFS_HPUX_ENV */
82 #include <afs/osi_inode.h>
85 #include <afs/afsutil.h>
86 #include <afs/fileutil.h>
87 #include <afs/procmgmt.h> /* signal(), kill(), wait(), etc. */
93 #include <afs/afssyscalls.h>
97 #include "partition.h"
98 #include "daemon_com.h"
100 #include "salvsync.h"
101 #include "viceinode.h"
103 #include "vol-salvage.h"
109 extern int ClientMode;
111 #if !defined(AFS_DEMAND_ATTACH_FS)
112 #error "online salvager only supported for demand attach fileserver"
113 #endif /* AFS_DEMAND_ATTACH_FS */
115 #if defined(AFS_NT40_ENV)
116 #error "online salvager not supported on NT"
117 #endif /* AFS_NT40_ENV */
119 /*@+fcnmacros +macrofcndecl@*/
121 #define afs_fopen fopen64
122 #else /* !O_LARGEFILE */
123 #define afs_fopen fopen
124 #endif /* !O_LARGEFILE */
125 /*@=fcnmacros =macrofcndecl@*/
129 static volatile int current_workers = 0;
130 static volatile struct rx_queue pending_q;
131 static pthread_mutex_t worker_lock;
132 static pthread_cond_t worker_cv;
134 static void * SalvageChildReaperThread(void *);
135 static int DoSalvageVolume(struct SalvageQueueNode * node, int slot);
137 static void SalvageServer(int argc, char **argv, struct logOptions *logopts);
138 static void SalvageClient(VolumeId vid, char * pname);
140 static int Reap_Child(char * prog, int * pid, int * status);
142 static void * SalvageLogCleanupThread(void *);
143 static void SalvageLogCleanup(int pid);
145 static void * SalvageLogScanningThread(void *);
146 static void ScanLogs(struct rx_queue *log_watch_queue);
148 struct cmdline_rock {
153 struct log_cleanup_node {
159 struct rx_queue queue_head;
160 pthread_cond_t queue_change_cv;
164 #define DEFAULT_PARALLELISM 4 /* allow 4 parallel salvage workers by default */
187 handleit(struct cmd_syndesc *opts, void *arock)
190 afs_int32 seenpart = 0, seenvol = 0;
192 struct cmdline_rock *rock = (struct cmdline_rock *)arock;
193 char *optstring = NULL;
194 struct logOptions logopts;
196 memset(&logopts, 0, sizeof(logopts));
198 #ifdef AFS_SGI_VNODE_GLUE
199 if (afs_init_kernel_config(-1) < 0) {
201 ("Can't determine NUMA configuration, not starting salvager.\n");
206 cmd_OptionAsFlag(opts, OPT_debug, &debug);
207 cmd_OptionAsFlag(opts, OPT_nowrite, &Testing);
208 cmd_OptionAsFlag(opts, OPT_inodes, &ListInodeOption);
209 cmd_OptionAsFlag(opts, OPT_oktozap, &OKToZap);
210 cmd_OptionAsFlag(opts, OPT_rootinodes, &ShowRootFiles);
211 cmd_OptionAsFlag(opts, OPT_salvagedirs, &RebuildDirs);
212 cmd_OptionAsFlag(opts, OPT_blockreads, &forceR);
213 if (cmd_OptionAsString(opts, OPT_parallel, &optstring) == 0) {
214 if (strncmp(optstring, "all", 3) == 0) {
217 if (strlen(optstring) != 0) {
218 Parallel = atoi(optstring);
221 if (Parallel > MAXPARALLEL) {
222 printf("Setting parallel salvages to maximum of %d \n",
224 Parallel = MAXPARALLEL;
230 Parallel = min(DEFAULT_PARALLELISM, MAXPARALLEL);
232 if (cmd_OptionAsString(opts, OPT_tmpdir, &optstring) == 0) {
234 dirp = opendir(optstring);
237 ("Can't open temporary placeholder dir %s; using current partition \n",
245 if (cmd_OptionAsString(opts, OPT_orphans, &optstring) == 0) {
247 orphans = ORPH_IGNORE;
248 else if (strcmp(optstring, "remove") == 0
249 || strcmp(optstring, "r") == 0)
250 orphans = ORPH_REMOVE;
251 else if (strcmp(optstring, "attach") == 0
252 || strcmp(optstring, "a") == 0)
253 orphans = ORPH_ATTACH;
259 if (cmd_OptionPresent(opts, OPT_syslog)) {
260 if (cmd_OptionPresent(opts, OPT_logfile)) {
261 fprintf(stderr, "Invalid options: -syslog and -logfile are exclusive.\n");
264 if (cmd_OptionPresent(opts, OPT_transarc_logs)) {
265 fprintf(stderr, "Invalid options: -syslog and -transarc-logs are exclusive.\n");
268 logopts.lopt_dest = logDest_syslog;
269 logopts.lopt_facility = LOG_DAEMON;
270 logopts.lopt_tag = "salvageserver";
271 cmd_OptionAsInt(opts, OPT_syslogfacility, &logopts.lopt_facility);
275 logopts.lopt_dest = logDest_file;
276 if (cmd_OptionPresent(opts, OPT_transarc_logs)) {
277 logopts.lopt_rotateOnOpen = 1;
278 logopts.lopt_rotateStyle = logRotate_old;
280 if (cmd_OptionPresent(opts, OPT_logfile))
281 cmd_OptionAsString(opts, OPT_logfile, (char**)&logopts.lopt_filename);
283 logopts.lopt_filename = AFSDIR_SERVER_SALSRVLOG_FILEPATH;
286 if (cmd_OptionPresent(opts, OPT_client)) {
287 if (cmd_OptionAsString(opts, OPT_partition, &optstring) == 0) {
289 strlcpy(pname, optstring, sizeof(pname));
293 if (cmd_OptionAsString(opts, OPT_volumeid, &optstring) == 0) {
297 vid_l = strtoul(optstring, &end, 10);
298 if (vid_l >= MAX_AFS_UINT32 || vid_l == ULONG_MAX || *end != '\0') {
299 printf("Invalid volume id specified; salvage aborted\n");
302 vid = (VolumeId)vid_l;
305 if (!seenpart || !seenvol) {
306 printf("You must specify '-partition' and '-volumeid' with the '-client' option\n");
310 SalvageClient(vid, pname);
312 } else { /* salvageserver mode */
313 SalvageServer(rock->argc, rock->argv, &logopts);
320 #include "AFS_component_version_number.c"
324 char *save_args[MAX_ARGS];
326 pthread_t main_thread;
330 main(int argc, char **argv)
332 struct cmd_syndesc *ts;
334 struct cmdline_rock arock;
338 * The following signal action for AIX is necessary so that in case of a
339 * crash (i.e. core is generated) we can include the user's data section
340 * in the core dump. Unfortunately, by default, only a partial core is
341 * generated which, in many cases, isn't too useful.
343 struct sigaction nsa;
345 sigemptyset(&nsa.sa_mask);
346 nsa.sa_handler = SIG_DFL;
347 nsa.sa_flags = SA_FULLDUMP;
348 sigaction(SIGABRT, &nsa, NULL);
349 sigaction(SIGSEGV, &nsa, NULL);
352 /* Initialize directory paths */
353 if (!(initAFSDirPath() & AFSDIR_SERVER_PATHS_OK)) {
355 ReportErrorEventAlt(AFSEVT_SVR_NO_INSTALL_DIR, 0, argv[0], 0);
357 fprintf(stderr, "%s: Unable to obtain AFS server directory.\n",
362 /* Default to binary mode for fopen() */
363 _set_fmode(_O_BINARY);
365 main_thread = pthread_self();
366 if (spawnDatap && spawnDataLen) {
367 /* This is a child per partition salvager. Don't setup log or
368 * try to lock the salvager lock.
370 if (nt_SetupPartitionSalvage(spawnDatap, spawnDataLen) < 0)
376 if (geteuid() != 0) {
377 printf("Salvager must be run as root.\n");
383 /* bad for normal help flag processing, but can do nada */
393 ts = cmd_CreateSyntax("initcmd", handleit, &arock, 0, "initialize the program");
394 cmd_AddParmAtOffset(ts, OPT_partition, "-partition", CMD_SINGLE,
395 CMD_OPTIONAL, "Name of partition to salvage");
396 cmd_AddParmAtOffset(ts, OPT_volumeid, "-volumeid", CMD_SINGLE, CMD_OPTIONAL,
397 "Volume Id to salvage");
398 cmd_AddParmAtOffset(ts, OPT_debug, "-debug", CMD_FLAG, CMD_OPTIONAL,
399 "Run in Debugging mode");
400 cmd_AddParmAtOffset(ts, OPT_nowrite, "-nowrite", CMD_FLAG, CMD_OPTIONAL,
401 "Run readonly/test mode");
402 cmd_AddParmAtOffset(ts, OPT_inodes, "-inodes", CMD_FLAG, CMD_OPTIONAL,
403 "Just list affected afs inodes - debugging flag");
404 cmd_AddParmAtOffset(ts, OPT_oktozap, "-oktozap", CMD_FLAG, CMD_OPTIONAL,
405 "Give permission to destroy bogus inodes/volumes - debugging flag");
406 cmd_AddParmAtOffset(ts, OPT_rootinodes, "-rootinodes", CMD_FLAG,
407 CMD_OPTIONAL, "Show inodes owned by root - debugging flag");
408 cmd_AddParmAtOffset(ts, OPT_salvagedirs, "-salvagedirs", CMD_FLAG,
409 CMD_OPTIONAL, "Force rebuild/salvage of all directories");
410 cmd_AddParmAtOffset(ts, OPT_blockreads, "-blockreads", CMD_FLAG,
411 CMD_OPTIONAL, "Read smaller blocks to handle IO/bad blocks");
412 cmd_AddParmAtOffset(ts, OPT_parallel, "-parallel", CMD_SINGLE, CMD_OPTIONAL,
413 "# of max parallel partition salvaging");
414 cmd_AddParmAtOffset(ts, OPT_tmpdir, "-tmpdir", CMD_SINGLE, CMD_OPTIONAL,
415 "Name of dir to place tmp files ");
416 cmd_AddParmAtOffset(ts, OPT_orphans, "-orphans", CMD_SINGLE, CMD_OPTIONAL,
417 "ignore | remove | attach");
420 cmd_AddParmAtOffset(ts, OPT_syslog, "-syslog", CMD_FLAG, CMD_OPTIONAL,
421 "Write salvage log to syslogs");
422 cmd_AddParmAtOffset(ts, OPT_syslogfacility, "-syslogfacility", CMD_SINGLE,
423 CMD_OPTIONAL, "Syslog facility number to use");
426 cmd_AddParmAtOffset(ts, OPT_client, "-client", CMD_FLAG, CMD_OPTIONAL,
427 "Use SALVSYNC to ask salvageserver to salvage a volume");
429 cmd_AddParmAtOffset(ts, OPT_logfile, "-logfile", CMD_SINGLE, CMD_OPTIONAL,
430 "Location of log file ");
432 cmd_AddParmAtOffset(ts, OPT_transarc_logs, "-transarc-logs", CMD_FLAG,
433 CMD_OPTIONAL, "enable Transarc style logging");
435 err = cmd_Dispatch(argc, argv);
437 AFS_UNREACHED(return 0);
441 SalvageClient(VolumeId vid, char * pname)
446 SALVSYNC_response_hdr sres;
447 VolumePackageOptions opts;
449 /* Send Log() messages to stderr in client mode. */
452 VOptDefaults(volumeUtility, &opts);
453 if (VInitVolumePackage2(volumeUtility, &opts)) {
454 /* VInitVolumePackage2 can fail on e.g. partition attachment errors,
455 * but we don't really care, since all we're doing is trying to use
457 fprintf(stderr, "errors encountered initializing volume package, but "
458 "trying to continue anyway\n");
460 SALVSYNC_clientInit();
462 code = SALVSYNC_SalvageVolume(vid, pname, SALVSYNC_SALVAGE, SALVSYNC_OPERATOR, 0, NULL);
463 if (code != SYNC_OK) {
467 res.payload.buf = (void *) &sres;
468 res.payload.len = sizeof(sres);
472 code = SALVSYNC_SalvageVolume(vid, pname, SALVSYNC_QUERY, SALVSYNC_WHATEVER, 0, &res);
473 if (code != SYNC_OK) {
476 switch (sres.state) {
477 case SALVSYNC_STATE_ERROR:
478 printf("salvageserver reports salvage ended in an error; check log files for more details\n");
479 case SALVSYNC_STATE_DONE:
480 case SALVSYNC_STATE_UNKNOWN:
484 SALVSYNC_clientFinis();
488 if (code == SYNC_DENIED) {
489 printf("salvageserver refused to salvage volume %u on partition %s\n",
491 } else if (code == SYNC_BAD_COMMAND) {
492 printf("SALVSYNC protocol mismatch; please make sure fileserver, volserver, salvageserver and salvager are same version\n");
493 } else if (code == SYNC_COM_ERROR) {
494 printf("SALVSYNC communications error\n");
496 SALVSYNC_clientFinis();
500 static int * child_slot;
503 SalvageServer(int argc, char **argv, struct logOptions *logopts)
506 struct SalvageQueueNode * node;
508 pthread_attr_t attrs;
510 VolumePackageOptions opts;
512 /* All entries to the log will be appended. Useful if there are
513 * multiple salvagers appending to the log.
518 Log("%s\n", cml_version_number);
519 LogCommandLine(argc, argv, "Online Salvage Server",
520 SalvageVersion, "Starting OpenAFS", Log);
521 /* Get and hold a lock for the duration of the salvage to make sure
522 * that no other salvage runs at the same time. The routine
523 * VInitVolumePackage2 (called below) makes sure that a file server or
524 * other volume utilities don't interfere with the salvage.
527 /* even demand attach online salvager
528 * still needs this because we don't want
529 * a stand-alone salvager to conflict with
530 * the salvager daemon */
531 ObtainSharedSalvageLock();
533 child_slot = calloc(Parallel, sizeof(int));
534 opr_Assert(child_slot != NULL);
536 /* initialize things */
537 VOptDefaults(salvageServer, &opts);
538 if (VInitVolumePackage2(salvageServer, &opts)) {
539 Log("Shutting down: errors encountered initializing volume package\n");
543 queue_Init(&pending_q);
544 queue_Init(&log_cleanup_queue);
545 opr_mutex_init(&worker_lock);
546 opr_cv_init(&worker_cv);
547 opr_cv_init(&log_cleanup_queue.queue_change_cv);
548 opr_Verify(pthread_attr_init(&attrs) == 0);
550 /* start up the reaper and log cleaner threads */
551 opr_Verify(pthread_attr_setdetachstate(&attrs,
552 PTHREAD_CREATE_DETACHED) == 0);
553 opr_Verify(pthread_create(&tid, &attrs,
554 &SalvageChildReaperThread, NULL) == 0);
555 opr_Verify(pthread_create(&tid, &attrs,
556 &SalvageLogCleanupThread, NULL) == 0);
557 opr_Verify(pthread_create(&tid, &attrs,
558 &SalvageLogScanningThread, NULL) == 0);
560 /* loop forever serving requests */
562 node = SALVSYNC_getWork();
563 opr_Assert(node != NULL);
565 Log("dispatching child to salvage volume %u...\n",
566 node->command.sop.parent);
570 for (slot = 0; slot < Parallel; slot++) {
571 if (!child_slot[slot])
574 opr_Assert (slot < Parallel);
580 ret = DoSalvageVolume(node, slot);
582 } else if (pid < 0) {
583 Log("failed to fork child worker process\n");
587 child_slot[slot] = pid;
591 opr_mutex_enter(&worker_lock);
594 /* let the reaper thread know another worker was spawned */
595 opr_cv_broadcast(&worker_cv);
597 /* if we're overquota, wait for the reaper */
598 while (current_workers >= Parallel) {
599 opr_cv_wait(&worker_cv, &worker_lock);
601 opr_mutex_exit(&worker_lock);
607 DoSalvageVolume(struct SalvageQueueNode * node, int slot)
609 char *filename = NULL;
610 struct logOptions logopts;
611 struct DiskPartition64 * partP;
613 /* do not allow further forking inside salvager */
617 * Do not attempt to close parent's log file handle as
618 * another thread may have held the lock when fork was
621 memset(&logopts, 0, sizeof(logopts));
622 logopts.lopt_dest = logDest_file;
623 logopts.lopt_rotateStyle = logRotate_none;
624 if (asprintf(&filename, "%s.%d",
625 AFSDIR_SERVER_SLVGLOG_FILEPATH, getpid()) < 0) {
626 fprintf(stderr, "out of memory\n");
629 logopts.lopt_filename = filename;
633 if (node->command.sop.parent <= 0) {
634 Log("salvageServer: invalid volume id specified; salvage aborted\n");
638 partP = VGetPartition(node->command.sop.partName, 0);
640 Log("salvageServer: Unknown or unmounted partition %s; salvage aborted\n",
641 node->command.sop.partName);
645 /* obtain a shared salvage lock in the child worker, so if the
646 * salvageserver restarts (and we continue), we will still hold a lock and
647 * prevent standalone salvagers from interfering */
648 ObtainSharedSalvageLock();
650 /* Salvage individual volume; don't notify fs */
651 SalvageFileSys1(partP, node->command.sop.parent);
659 SalvageChildReaperThread(void * args)
661 int slot, pid, status;
662 struct log_cleanup_node * cleanup;
664 opr_mutex_enter(&worker_lock);
666 /* loop reaping our children */
668 /* wait() won't block unless we have children, so
669 * block on the cond var if we're childless */
670 while (current_workers == 0) {
671 opr_cv_wait(&worker_cv, &worker_lock);
674 opr_mutex_exit(&worker_lock);
676 cleanup = malloc(sizeof(struct log_cleanup_node));
678 while (Reap_Child("salvageserver", &pid, &status) < 0) {
679 /* try to prevent livelock if something goes wrong */
684 for (slot = 0; slot < Parallel; slot++) {
685 if (child_slot[slot] == pid)
688 opr_Assert(slot < Parallel);
689 child_slot[slot] = 0;
692 SALVSYNC_doneWorkByPid(pid, status);
694 opr_mutex_enter(&worker_lock);
698 queue_Append(&log_cleanup_queue, cleanup);
699 opr_cv_signal(&log_cleanup_queue.queue_change_cv);
702 /* ok, we've reaped a child */
704 opr_cv_broadcast(&worker_cv);
707 AFS_UNREACHED(return(NULL));
711 Reap_Child(char *prog, int * pid, int * status)
718 if (WCOREDUMP(*status))
719 Log("\"%s\" core dumped!\n", prog);
720 if ((WIFSIGNALED(*status) != 0) ||
721 ((WEXITSTATUS(*status) != 0) &&
722 (WEXITSTATUS(*status) != SALSRV_EXIT_VOLGROUP_LINK)))
723 Log("\"%s\" (pid=%d) terminated abnormally!\n", prog, ret);
725 Log("wait returned -1\n");
731 * thread to combine salvager child logs
732 * back into the main salvageserver log
735 SalvageLogCleanupThread(void * arg)
737 struct log_cleanup_node * cleanup;
739 opr_mutex_enter(&worker_lock);
742 while (queue_IsEmpty(&log_cleanup_queue)) {
743 opr_cv_wait(&log_cleanup_queue.queue_change_cv, &worker_lock);
746 while (queue_IsNotEmpty(&log_cleanup_queue)) {
747 cleanup = queue_First(&log_cleanup_queue, log_cleanup_node);
748 queue_Remove(cleanup);
749 opr_mutex_exit(&worker_lock);
750 SalvageLogCleanup(cleanup->pid);
752 opr_mutex_enter(&worker_lock);
756 AFS_UNREACHED(opr_mutex_exit(&worker_lock));
757 AFS_UNREACHED(return(NULL));
760 #define LOG_XFER_BUF_SIZE 65536
762 SalvageLogCleanup(int pid)
768 if (asprintf(&fn, "%s.%d", AFSDIR_SERVER_SLVGLOG_FILEPATH, pid) < 0) {
769 Log("Unable to write child log: out of memory\n");
773 buf = calloc(1, LOG_XFER_BUF_SIZE);
775 Log("Unable to write child log: out of memory\n");
779 pidlog = open(fn, O_RDONLY);
784 len = read(pidlog, buf, LOG_XFER_BUF_SIZE);
786 WriteLogBuffer(buf, len);
787 len = read(pidlog, buf, LOG_XFER_BUF_SIZE);
797 /* wake up every five minutes to see if a non-child salvage has finished */
798 #define SALVAGE_SCAN_POLL_INTERVAL 300
801 * Thread to look for SalvageLog.$pid files that are not from our child
802 * worker salvagers, and notify SalvageLogCleanupThread to clean them
803 * up. This can happen if we restart during salvages, or the
804 * salvageserver crashes or something.
808 * @return always NULL
811 SalvageLogScanningThread(void * arg)
813 struct rx_queue log_watch_queue;
817 queue_Init(&log_watch_queue);
819 prefix_len = asprintf(&prefix, "%s.", AFSDIR_SLVGLOG_FILE);
820 if (prefix_len >= 0) {
824 dp = opendir(AFSDIR_LOGS_DIR);
827 while ((dirp = readdir(dp)) != NULL) {
829 struct log_cleanup_node *cleanup;
832 if (strncmp(dirp->d_name, prefix, prefix_len) != 0) {
833 /* not a salvage logfile; skip */
838 pid = strtol(dirp->d_name + prefix_len, NULL, 10);
841 /* file is SalvageLog.<something> but <something> isn't
847 for (i = 0; i < Parallel; ++i) {
848 if (pid == child_slot[i]) {
854 /* this pid is one of our children, so the reaper thread
855 * will take care of it; skip */
859 cleanup = malloc(sizeof(struct log_cleanup_node));
862 queue_Append(&log_watch_queue, cleanup);
868 ScanLogs(&log_watch_queue);
870 while (queue_IsNotEmpty(&log_watch_queue)) {
871 sleep(SALVAGE_SCAN_POLL_INTERVAL);
872 ScanLogs(&log_watch_queue);
879 * look through log_watch_queue, and if any processes are not still
880 * running, hand them off to the SalvageLogCleanupThread
882 * @param log_watch_queue a queue of PIDs that we should clean up if
886 ScanLogs(struct rx_queue *log_watch_queue)
888 struct log_cleanup_node *cleanup, *next;
890 opr_mutex_enter(&worker_lock);
892 for (queue_Scan(log_watch_queue, cleanup, next, log_cleanup_node)) {
893 /* if a process is still running, assume it's the salvage process
894 * still going, and keep waiting for it */
895 if (kill(cleanup->pid, 0) < 0 && errno == ESRCH) {
896 queue_Remove(cleanup);
897 queue_Append(&log_cleanup_queue, cleanup);
898 opr_cv_signal(&log_cleanup_queue.queue_change_cv);
902 opr_mutex_exit(&worker_lock);