2 * Copyright 2006-2007, Sine Nomine Associates and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
12 * online salvager daemon
15 /* Main program file. Define globals. */
18 #include <afsconfig.h>
19 #include <afs/param.h>
23 #ifdef HAVE_SYS_FILE_H
28 #include <WINNT/afsevent.h>
32 #define WCOREDUMP(x) ((x) & 0200)
36 #include <afs/afsint.h>
37 #include <afs/afs_assert.h>
39 #if !defined(AFS_SGI_ENV) && !defined(AFS_NT40_ENV)
40 #if defined(AFS_VFSINCL_ENV)
41 #include <sys/vnode.h>
43 #include <sys/fs/ufs_inode.h>
45 #if defined(AFS_DARWIN_ENV) || defined(AFS_XBSD_ENV)
46 #include <ufs/ufs/dinode.h>
47 #include <ufs/ffs/fs.h>
49 #include <ufs/inode.h>
52 #else /* AFS_VFSINCL_ENV */
54 #include <ufs/inode.h>
55 #else /* AFS_OSF_ENV */
56 #if !defined(AFS_LINUX20_ENV) && !defined(AFS_XBSD_ENV) && !defined(AFS_ARM_DARWIN_ENV)
57 #include <sys/inode.h>
60 #endif /* AFS_VFSINCL_ENV */
61 #endif /* AFS_SGI_ENV */
64 #include <sys/lockf.h>
67 #include <checklist.h>
69 #if defined(AFS_SGI_ENV)
72 #if defined(AFS_SUN_ENV) || defined(AFS_SUN5_ENV)
74 #include <sys/mnttab.h>
75 #include <sys/mntent.h>
80 #endif /* AFS_SGI_ENV */
81 #endif /* AFS_HPUX_ENV */
85 #include <afs/osi_inode.h>
88 #include <afs/afsutil.h>
89 #include <afs/fileutil.h>
90 #include <afs/procmgmt.h> /* signal(), kill(), wait(), etc. */
96 #include <afs/afssyscalls.h>
100 #include "partition.h"
101 #include "daemon_com.h"
103 #include "salvsync.h"
104 #include "viceinode.h"
106 #include "vol-salvage.h"
113 #if !defined(AFS_DEMAND_ATTACH_FS)
114 #error "online salvager only supported for demand attach fileserver"
115 #endif /* AFS_DEMAND_ATTACH_FS */
117 #if defined(AFS_NT40_ENV)
118 #error "online salvager not supported on NT"
119 #endif /* AFS_NT40_ENV */
121 /*@+fcnmacros +macrofcndecl@*/
123 #define afs_fopen fopen64
124 #else /* !O_LARGEFILE */
125 #define afs_fopen fopen
126 #endif /* !O_LARGEFILE */
127 /*@=fcnmacros =macrofcndecl@*/
131 static volatile int current_workers = 0;
132 static volatile struct rx_queue pending_q;
133 static pthread_mutex_t worker_lock;
134 static pthread_cond_t worker_cv;
136 static void * SalvageChildReaperThread(void *);
137 static int DoSalvageVolume(struct SalvageQueueNode * node, int slot);
139 static void SalvageServer(int argc, char **argv);
140 static void SalvageClient(VolumeId vid, char * pname);
142 static int Reap_Child(char * prog, int * pid, int * status);
144 static void * SalvageLogCleanupThread(void *);
145 static int SalvageLogCleanup(int pid);
147 static void * SalvageLogScanningThread(void *);
148 static void ScanLogs(struct rx_queue *log_watch_queue);
150 struct cmdline_rock {
155 struct log_cleanup_node {
161 struct rx_queue queue_head;
162 pthread_cond_t queue_change_cv;
166 #define DEFAULT_PARALLELISM 4 /* allow 4 parallel salvage workers by default */
169 handleit(struct cmd_syndesc *as, void *arock)
172 char pname[100], *temp;
173 afs_int32 seenpart = 0, seenvol = 0;
175 struct cmdline_rock *rock = (struct cmdline_rock *)arock;
177 #ifdef AFS_SGI_VNODE_GLUE
178 if (afs_init_kernel_config(-1) < 0) {
180 ("Can't determine NUMA configuration, not starting salvager.\n");
185 if (as->parms[2].items) /* -debug */
187 if (as->parms[3].items) /* -nowrite */
189 if (as->parms[4].items) /* -inodes */
191 if (as->parms[5].items) /* -oktozap */
193 if (as->parms[6].items) /* -rootinodes */
195 if (as->parms[8].items) /* -ForceReads */
197 if ((ti = as->parms[9].items)) { /* -Parallel # */
199 if (strncmp(temp, "all", 3) == 0) {
203 if (strlen(temp) != 0) {
204 Parallel = atoi(temp);
207 if (Parallel > MAXPARALLEL) {
208 printf("Setting parallel salvages to maximum of %d \n",
210 Parallel = MAXPARALLEL;
214 Parallel = MIN(DEFAULT_PARALLELISM, MAXPARALLEL);
216 if ((ti = as->parms[10].items)) { /* -tmpdir */
220 dirp = opendir(tmpdir);
223 ("Can't open temporary placeholder dir %s; using current partition \n",
229 if ((ti = as->parms[11].items)) /* -showlog */
231 if ((ti = as->parms[12].items)) { /* -orphans */
233 orphans = ORPH_IGNORE;
234 else if (strcmp(ti->data, "remove") == 0
235 || strcmp(ti->data, "r") == 0)
236 orphans = ORPH_REMOVE;
237 else if (strcmp(ti->data, "attach") == 0
238 || strcmp(ti->data, "a") == 0)
239 orphans = ORPH_ATTACH;
241 #ifndef AFS_NT40_ENV /* ignore options on NT */
242 if ((ti = as->parms[13].items)) { /* -syslog */
246 if ((ti = as->parms[14].items)) { /* -syslogfacility */
247 useSyslogFacility = atoi(ti->data);
250 if ((ti = as->parms[15].items)) { /* -datelogs */
251 TimeStampLogFile((char *)AFSDIR_SERVER_SALSRVLOG_FILEPATH);
255 if ((ti = as->parms[16].items)) { /* -client */
256 if ((ti = as->parms[0].items)) { /* -partition */
258 strlcpy(pname, ti->data, sizeof(pname));
260 if ((ti = as->parms[1].items)) { /* -volumeid */
264 vid_l = strtoul(ti->data, &end, 10);
265 if (vid_l >= MAX_AFS_UINT32 || vid_l == ULONG_MAX || *end != '\0') {
266 printf("Invalid volume id specified; salvage aborted\n");
269 vid = (VolumeId)vid_l;
273 printf("-showlog does not work with -client\n");
277 if (!seenpart || !seenvol) {
278 printf("You must specify '-partition' and '-volumeid' with the '-client' option\n");
282 SalvageClient(vid, pname);
284 } else { /* salvageserver mode */
285 SalvageServer(rock->argc, rock->argv);
292 #include "AFS_component_version_number.c"
296 char *save_args[MAX_ARGS];
298 pthread_t main_thread;
302 main(int argc, char **argv)
304 struct cmd_syndesc *ts;
306 struct cmdline_rock arock;
310 * The following signal action for AIX is necessary so that in case of a
311 * crash (i.e. core is generated) we can include the user's data section
312 * in the core dump. Unfortunately, by default, only a partial core is
313 * generated which, in many cases, isn't too useful.
315 struct sigaction nsa;
317 sigemptyset(&nsa.sa_mask);
318 nsa.sa_handler = SIG_DFL;
319 nsa.sa_flags = SA_FULLDUMP;
320 sigaction(SIGABRT, &nsa, NULL);
321 sigaction(SIGSEGV, &nsa, NULL);
324 /* Initialize directory paths */
325 if (!(initAFSDirPath() & AFSDIR_SERVER_PATHS_OK)) {
327 ReportErrorEventAlt(AFSEVT_SVR_NO_INSTALL_DIR, 0, argv[0], 0);
329 fprintf(stderr, "%s: Unable to obtain AFS server directory.\n",
334 /* Default to binary mode for fopen() */
335 _set_fmode(_O_BINARY);
337 main_thread = pthread_self();
338 if (spawnDatap && spawnDataLen) {
339 /* This is a child per partition salvager. Don't setup log or
340 * try to lock the salvager lock.
342 if (nt_SetupPartitionSalvage(spawnDatap, spawnDataLen) < 0)
348 if (geteuid() != 0) {
349 printf("Salvager must be run as root.\n");
355 /* bad for normal help flag processing, but can do nada */
364 ts = cmd_CreateSyntax("initcmd", handleit, &arock, "initialize the program");
365 cmd_AddParm(ts, "-partition", CMD_SINGLE, CMD_OPTIONAL,
366 "Name of partition to salvage");
367 cmd_AddParm(ts, "-volumeid", CMD_SINGLE, CMD_OPTIONAL,
368 "Volume Id to salvage");
369 cmd_AddParm(ts, "-debug", CMD_FLAG, CMD_OPTIONAL,
370 "Run in Debugging mode");
371 cmd_AddParm(ts, "-nowrite", CMD_FLAG, CMD_OPTIONAL,
372 "Run readonly/test mode");
373 cmd_AddParm(ts, "-inodes", CMD_FLAG, CMD_OPTIONAL,
374 "Just list affected afs inodes - debugging flag");
375 cmd_AddParm(ts, "-oktozap", CMD_FLAG, CMD_OPTIONAL,
376 "Give permission to destroy bogus inodes/volumes - debugging flag");
377 cmd_AddParm(ts, "-rootinodes", CMD_FLAG, CMD_OPTIONAL,
378 "Show inodes owned by root - debugging flag");
379 cmd_AddParm(ts, "-salvagedirs", CMD_FLAG, CMD_OPTIONAL,
380 "Force rebuild/salvage of all directories");
381 cmd_AddParm(ts, "-blockreads", CMD_FLAG, CMD_OPTIONAL,
382 "Read smaller blocks to handle IO/bad blocks");
383 cmd_AddParm(ts, "-parallel", CMD_SINGLE, CMD_OPTIONAL,
384 "# of max parallel partition salvaging");
385 cmd_AddParm(ts, "-tmpdir", CMD_SINGLE, CMD_OPTIONAL,
386 "Name of dir to place tmp files ");
387 cmd_AddParm(ts, "-showlog", CMD_FLAG, CMD_OPTIONAL,
388 "Show log file upon completion");
389 cmd_AddParm(ts, "-orphans", CMD_SINGLE, CMD_OPTIONAL,
390 "ignore | remove | attach");
392 /* note - syslog isn't avail on NT, but if we make it conditional, have
393 * to deal with screwy offsets for cmd params */
394 cmd_AddParm(ts, "-syslog", CMD_FLAG, CMD_OPTIONAL,
395 "Write salvage log to syslogs");
396 cmd_AddParm(ts, "-syslogfacility", CMD_SINGLE, CMD_OPTIONAL,
397 "Syslog facility number to use");
398 cmd_AddParm(ts, "-datelogs", CMD_FLAG, CMD_OPTIONAL,
399 "Include timestamp in logfile filename");
401 cmd_AddParm(ts, "-client", CMD_FLAG, CMD_OPTIONAL,
402 "Use SALVSYNC to ask salvageserver to salvage a volume");
404 err = cmd_Dispatch(argc, argv);
406 return 0; /* not reached */
410 SalvageClient(VolumeId vid, char * pname)
415 SALVSYNC_response_hdr sres;
416 VolumePackageOptions opts;
418 VOptDefaults(volumeUtility, &opts);
419 if (VInitVolumePackage2(volumeUtility, &opts)) {
420 /* VInitVolumePackage2 can fail on e.g. partition attachment errors,
421 * but we don't really care, since all we're doing is trying to use
423 fprintf(stderr, "errors encountered initializing volume package, but "
424 "trying to continue anyway\n");
426 SALVSYNC_clientInit();
428 code = SALVSYNC_SalvageVolume(vid, pname, SALVSYNC_SALVAGE, SALVSYNC_OPERATOR, 0, NULL);
429 if (code != SYNC_OK) {
433 res.payload.buf = (void *) &sres;
434 res.payload.len = sizeof(sres);
438 code = SALVSYNC_SalvageVolume(vid, pname, SALVSYNC_QUERY, SALVSYNC_WHATEVER, 0, &res);
439 if (code != SYNC_OK) {
442 switch (sres.state) {
443 case SALVSYNC_STATE_ERROR:
444 printf("salvageserver reports salvage ended in an error; check log files for more details\n");
445 case SALVSYNC_STATE_DONE:
446 case SALVSYNC_STATE_UNKNOWN:
450 SALVSYNC_clientFinis();
454 if (code == SYNC_DENIED) {
455 printf("salvageserver refused to salvage volume %u on partition %s\n",
457 } else if (code == SYNC_BAD_COMMAND) {
458 printf("SALVSYNC protocol mismatch; please make sure fileserver, volserver, salvageserver and salvager are same version\n");
459 } else if (code == SYNC_COM_ERROR) {
460 printf("SALVSYNC communications error\n");
462 SALVSYNC_clientFinis();
466 static int * child_slot;
469 SalvageServer(int argc, char **argv)
472 struct SalvageQueueNode * node;
474 pthread_attr_t attrs;
476 VolumePackageOptions opts;
478 /* All entries to the log will be appended. Useful if there are
479 * multiple salvagers appending to the log.
482 CheckLogFile((char *)AFSDIR_SERVER_SALSRVLOG_FILEPATH);
484 #ifdef AFS_LINUX20_ENV
485 fcntl(fileno(logFile), F_SETFL, O_APPEND); /* Isn't this redundant? */
487 fcntl(fileno(logFile), F_SETFL, FAPPEND); /* Isn't this redundant? */
492 fprintf(logFile, "%s\n", cml_version_number);
493 LogCommandLine(argc, argv, "Online Salvage Server",
494 SalvageVersion, "Starting OpenAFS", Log);
495 /* Get and hold a lock for the duration of the salvage to make sure
496 * that no other salvage runs at the same time. The routine
497 * VInitVolumePackage2 (called below) makes sure that a file server or
498 * other volume utilities don't interfere with the salvage.
501 /* even demand attach online salvager
502 * still needs this because we don't want
503 * a stand-alone salvager to conflict with
504 * the salvager daemon */
505 ObtainSharedSalvageLock();
507 child_slot = (int *) malloc(Parallel * sizeof(int));
508 osi_Assert(child_slot != NULL);
509 memset(child_slot, 0, Parallel * sizeof(int));
511 /* initialize things */
512 VOptDefaults(salvageServer, &opts);
513 if (VInitVolumePackage2(salvageServer, &opts)) {
514 Log("Shutting down: errors encountered initializing volume package\n");
518 queue_Init(&pending_q);
519 queue_Init(&log_cleanup_queue);
520 MUTEX_INIT(&worker_lock, "worker", MUTEX_DEFAULT, 0);
521 CV_INIT(&worker_cv, "worker", CV_DEFAULT, 0);
522 CV_INIT(&log_cleanup_queue.queue_change_cv, "queuechange", CV_DEFAULT, 0);
523 osi_Assert(pthread_attr_init(&attrs) == 0);
525 /* start up the reaper and log cleaner threads */
526 osi_Assert(pthread_attr_setdetachstate(&attrs, PTHREAD_CREATE_DETACHED) == 0);
527 osi_Assert(pthread_create(&tid,
529 &SalvageChildReaperThread,
531 osi_Assert(pthread_create(&tid,
533 &SalvageLogCleanupThread,
535 osi_Assert(pthread_create(&tid,
537 &SalvageLogScanningThread,
540 /* loop forever serving requests */
542 node = SALVSYNC_getWork();
543 osi_Assert(node != NULL);
545 Log("dispatching child to salvage volume %u...\n",
546 node->command.sop.parent);
550 for (slot = 0; slot < Parallel; slot++) {
551 if (!child_slot[slot])
554 osi_Assert (slot < Parallel);
560 ret = DoSalvageVolume(node, slot);
562 } else if (pid < 0) {
563 Log("failed to fork child worker process\n");
567 child_slot[slot] = pid;
571 MUTEX_ENTER(&worker_lock);
574 /* let the reaper thread know another worker was spawned */
575 CV_BROADCAST(&worker_cv);
577 /* if we're overquota, wait for the reaper */
578 while (current_workers >= Parallel) {
579 CV_WAIT(&worker_cv, &worker_lock);
581 MUTEX_EXIT(&worker_lock);
587 DoSalvageVolume(struct SalvageQueueNode * node, int slot)
589 char childLog[AFSDIR_PATH_MAX];
590 struct DiskPartition64 * partP;
592 /* do not allow further forking inside salvager */
595 /* do not attempt to close parent's logFile handle as
596 * another thread may have held the lock on the FILE
597 * structure when fork was called! */
599 snprintf(childLog, sizeof(childLog), "%s.%d",
600 AFSDIR_SERVER_SLVGLOG_FILEPATH, getpid());
602 logFile = afs_fopen(childLog, "a");
603 if (!logFile) { /* still nothing, use stdout */
608 if (node->command.sop.parent <= 0) {
609 Log("salvageServer: invalid volume id specified; salvage aborted\n");
613 partP = VGetPartition(node->command.sop.partName, 0);
615 Log("salvageServer: Unknown or unmounted partition %s; salvage aborted\n",
616 node->command.sop.partName);
620 /* obtain a shared salvage lock in the child worker, so if the
621 * salvageserver restarts (and we continue), we will still hold a lock and
622 * prevent standalone salvagers from interfering */
623 ObtainSharedSalvageLock();
625 /* Salvage individual volume; don't notify fs */
626 SalvageFileSys1(partP, node->command.sop.parent);
634 SalvageChildReaperThread(void * args)
636 int slot, pid, status;
637 struct log_cleanup_node * cleanup;
639 MUTEX_ENTER(&worker_lock);
641 /* loop reaping our children */
643 /* wait() won't block unless we have children, so
644 * block on the cond var if we're childless */
645 while (current_workers == 0) {
646 CV_WAIT(&worker_cv, &worker_lock);
649 MUTEX_EXIT(&worker_lock);
651 cleanup = (struct log_cleanup_node *) malloc(sizeof(struct log_cleanup_node));
653 while (Reap_Child("salvageserver", &pid, &status) < 0) {
654 /* try to prevent livelock if something goes wrong */
659 for (slot = 0; slot < Parallel; slot++) {
660 if (child_slot[slot] == pid)
663 osi_Assert(slot < Parallel);
664 child_slot[slot] = 0;
667 SALVSYNC_doneWorkByPid(pid, status);
669 MUTEX_ENTER(&worker_lock);
673 queue_Append(&log_cleanup_queue, cleanup);
674 CV_SIGNAL(&log_cleanup_queue.queue_change_cv);
677 /* ok, we've reaped a child */
679 CV_BROADCAST(&worker_cv);
686 Reap_Child(char *prog, int * pid, int * status)
693 if (WCOREDUMP(*status))
694 Log("\"%s\" core dumped!\n", prog);
695 if ((WIFSIGNALED(*status) != 0) ||
696 ((WEXITSTATUS(*status) != 0) &&
697 (WEXITSTATUS(*status) != SALSRV_EXIT_VOLGROUP_LINK)))
698 Log("\"%s\" (pid=%d) terminated abnormally!\n", prog, ret);
700 Log("wait returned -1\n");
706 * thread to combine salvager child logs
707 * back into the main salvageserver log
710 SalvageLogCleanupThread(void * arg)
712 struct log_cleanup_node * cleanup;
714 MUTEX_ENTER(&worker_lock);
717 while (queue_IsEmpty(&log_cleanup_queue)) {
718 CV_WAIT(&log_cleanup_queue.queue_change_cv, &worker_lock);
721 while (queue_IsNotEmpty(&log_cleanup_queue)) {
722 cleanup = queue_First(&log_cleanup_queue, log_cleanup_node);
723 queue_Remove(cleanup);
724 MUTEX_EXIT(&worker_lock);
725 SalvageLogCleanup(cleanup->pid);
727 MUTEX_ENTER(&worker_lock);
731 MUTEX_EXIT(&worker_lock);
735 #define LOG_XFER_BUF_SIZE 65536
737 SalvageLogCleanup(int pid)
740 char fn[AFSDIR_PATH_MAX];
741 static char buf[LOG_XFER_BUF_SIZE];
743 snprintf(fn, sizeof(fn), "%s.%d",
744 AFSDIR_SERVER_SLVGLOG_FILEPATH, pid);
747 pidlog = open(fn, O_RDONLY);
752 len = read(pidlog, buf, LOG_XFER_BUF_SIZE);
754 fwrite(buf, len, 1, logFile);
755 len = read(pidlog, buf, LOG_XFER_BUF_SIZE);
763 /* wake up every five minutes to see if a non-child salvage has finished */
764 #define SALVAGE_SCAN_POLL_INTERVAL 300
767 * Thread to look for SalvageLog.$pid files that are not from our child
768 * worker salvagers, and notify SalvageLogCleanupThread to clean them
769 * up. This can happen if we restart during salvages, or the
770 * salvageserver crashes or something.
774 * @return always NULL
777 SalvageLogScanningThread(void * arg)
779 struct rx_queue log_watch_queue;
781 queue_Init(&log_watch_queue);
786 char prefix[AFSDIR_PATH_MAX];
789 snprintf(prefix, sizeof(prefix), "%s.", AFSDIR_SLVGLOG_FILE);
790 prefix_len = strlen(prefix);
792 dp = opendir(AFSDIR_LOGS_DIR);
795 while ((dirp = readdir(dp)) != NULL) {
797 struct log_cleanup_node *cleanup;
800 if (strncmp(dirp->d_name, prefix, prefix_len) != 0) {
801 /* not a salvage logfile; skip */
806 pid = strtol(dirp->d_name + prefix_len, NULL, 10);
809 /* file is SalvageLog.<something> but <something> isn't
815 for (i = 0; i < Parallel; ++i) {
816 if (pid == child_slot[i]) {
822 /* this pid is one of our children, so the reaper thread
823 * will take care of it; skip */
828 (struct log_cleanup_node *) malloc(sizeof(struct log_cleanup_node));
831 queue_Append(&log_watch_queue, cleanup);
837 ScanLogs(&log_watch_queue);
839 while (queue_IsNotEmpty(&log_watch_queue)) {
840 sleep(SALVAGE_SCAN_POLL_INTERVAL);
841 ScanLogs(&log_watch_queue);
848 * look through log_watch_queue, and if any processes are not still
849 * running, hand them off to the SalvageLogCleanupThread
851 * @param log_watch_queue a queue of PIDs that we should clean up if
855 ScanLogs(struct rx_queue *log_watch_queue)
857 struct log_cleanup_node *cleanup, *next;
859 MUTEX_ENTER(&worker_lock);
861 for (queue_Scan(log_watch_queue, cleanup, next, log_cleanup_node)) {
862 /* if a process is still running, assume it's the salvage process
863 * still going, and keep waiting for it */
864 if (kill(cleanup->pid, 0) < 0 && errno == ESRCH) {
865 queue_Remove(cleanup);
866 queue_Append(&log_cleanup_queue, cleanup);
867 CV_SIGNAL(&log_cleanup_queue.queue_change_cv);
871 MUTEX_EXIT(&worker_lock);