2 * Copyright 2006, Sine Nomine Associates and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
12 * online salvager daemon
15 /* Main program file. Define globals. */
18 #include <afsconfig.h>
19 #include <afs/param.h>
33 #include <WINNT/afsevent.h>
35 #include <sys/param.h>
39 #endif /* ITIMER_REAL */
41 #if defined(AFS_AIX_ENV) || defined(AFS_SUN4_ENV)
42 #define WCOREDUMP(x) (x & 0200)
45 #include <afs/afsint.h>
46 #include <afs/assert.h>
47 #if !defined(AFS_SGI_ENV) && !defined(AFS_NT40_ENV)
48 #if defined(AFS_VFSINCL_ENV)
49 #include <sys/vnode.h>
51 #include <sys/fs/ufs_inode.h>
53 #if defined(AFS_DARWIN_ENV) || defined(AFS_XBSD_ENV)
54 #include <ufs/ufs/dinode.h>
55 #include <ufs/ffs/fs.h>
57 #include <ufs/inode.h>
60 #else /* AFS_VFSINCL_ENV */
62 #include <ufs/inode.h>
63 #else /* AFS_OSF_ENV */
64 #if !defined(AFS_LINUX20_ENV) && !defined(AFS_XBSD_ENV)
65 #include <sys/inode.h>
68 #endif /* AFS_VFSINCL_ENV */
69 #endif /* AFS_SGI_ENV */
72 #include <sys/lockf.h>
76 #include <checklist.h>
78 #if defined(AFS_SGI_ENV)
83 #if defined(AFS_SUN_ENV) || defined(AFS_SUN5_ENV)
86 #include <sys/mnttab.h>
87 #include <sys/mntent.h>
92 #endif /* AFS_SGI_ENV */
93 #endif /* AFS_HPUX_ENV */
98 #include <afs/osi_inode.h>
101 #include <afs/afsutil.h>
102 #include <afs/fileutil.h>
103 #include <afs/procmgmt.h> /* signal(), kill(), wait(), etc. */
111 #include <afs/afssyscalls.h>
115 #include "partition.h"
116 #include "daemon_com.h"
118 #include "salvsync.h"
119 #include "viceinode.h"
121 #include "volinodes.h" /* header magic number, etc. stuff */
122 #include "vol-salvage.h"
128 #if !defined(AFS_DEMAND_ATTACH_FS)
129 #error "online salvager only supported for demand attach fileserver"
130 #endif /* AFS_DEMAND_ATTACH_FS */
132 #if defined(AFS_NT40_ENV)
133 #error "online salvager not supported on NT"
134 #endif /* AFS_NT40_ENV */
137 /* Forward declarations */
138 /*@printflike@*/ void Log(const char *format, ...);
139 /*@printflike@*/ void Abort(const char *format, ...);
142 /*@+fcnmacros +macrofcndecl@*/
144 #define afs_fopen fopen64
145 #else /* !O_LARGEFILE */
146 #define afs_fopen fopen
147 #endif /* !O_LARGEFILE */
148 /*@=fcnmacros =macrofcndecl@*/
152 static volatile int current_workers = 0;
153 static volatile struct rx_queue pending_q;
154 static pthread_mutex_t worker_lock;
155 static pthread_cond_t worker_cv;
157 static void * SalvageChildReaperThread(void *);
158 static int DoSalvageVolume(struct SalvageQueueNode * node, int slot);
160 static void SalvageServer(void);
161 static void SalvageClient(VolumeId vid, char * pname);
163 static int Reap_Child(char * prog, int * pid, int * status);
165 static void * SalvageLogCleanupThread(void *);
166 static int SalvageLogCleanup(int pid);
168 struct log_cleanup_node {
174 struct rx_queue queue_head;
175 pthread_cond_t queue_change_cv;
179 #define DEFAULT_PARALLELISM 4 /* allow 4 parallel salvage workers by default */
182 handleit(struct cmd_syndesc *as)
184 register struct cmd_item *ti;
185 char pname[100], *temp;
186 afs_int32 seenpart = 0, seenvol = 0, vid = 0, seenany = 0;
187 struct DiskPartition *partP;
190 #ifdef AFS_SGI_VNODE_GLUE
191 if (afs_init_kernel_config(-1) < 0) {
193 ("Can't determine NUMA configuration, not starting salvager.\n");
198 if (as->parms[2].items) /* -debug */
200 if (as->parms[3].items) /* -nowrite */
202 if (as->parms[4].items) /* -inodes */
204 if (as->parms[5].items) /* -oktozap */
206 if (as->parms[6].items) /* -rootinodes */
208 if (as->parms[8].items) /* -ForceReads */
210 if ((ti = as->parms[9].items)) { /* -Parallel # */
212 if (strncmp(temp, "all", 3) == 0) {
216 if (strlen(temp) != 0) {
217 Parallel = atoi(temp);
220 if (Parallel > MAXPARALLEL) {
221 printf("Setting parallel salvages to maximum of %d \n",
223 Parallel = MAXPARALLEL;
227 Parallel = MIN(DEFAULT_PARALLELISM, MAXPARALLEL);
229 if ((ti = as->parms[10].items)) { /* -tmpdir */
233 dirp = opendir(tmpdir);
236 ("Can't open temporary placeholder dir %s; using current partition \n",
242 if ((ti = as->parms[11].items)) /* -showlog */
244 if ((ti = as->parms[12].items)) { /* -orphans */
246 orphans = ORPH_IGNORE;
247 else if (strcmp(ti->data, "remove") == 0
248 || strcmp(ti->data, "r") == 0)
249 orphans = ORPH_REMOVE;
250 else if (strcmp(ti->data, "attach") == 0
251 || strcmp(ti->data, "a") == 0)
252 orphans = ORPH_ATTACH;
254 #ifndef AFS_NT40_ENV /* ignore options on NT */
255 if ((ti = as->parms[13].items)) { /* -syslog */
259 if ((ti = as->parms[14].items)) { /* -syslogfacility */
260 useSyslogFacility = atoi(ti->data);
263 if ((ti = as->parms[15].items)) { /* -datelogs */
264 TimeStampLogFile(AFSDIR_SERVER_SALSRVLOG_FILEPATH);
268 if ((ti = as->parms[16].items)) { /* -client */
269 if ((ti = as->parms[0].items)) { /* -partition */
271 strlcpy(pname, ti->data, sizeof(pname));
273 if ((ti = as->parms[1].items)) { /* -volumeid */
275 vid = atoi(ti->data);
278 if (!seenpart || !seenvol) {
279 printf("You must specify '-partition' and '-volumeid' with the '-client' option\n");
283 SalvageClient(vid, pname);
285 } else { /* salvageserver mode */
293 #include "AFS_component_version_number.c"
297 char *save_args[MAX_ARGS];
299 pthread_t main_thread;
302 static char commandLine[150];
305 main(int argc, char **argv)
307 struct cmd_syndesc *ts;
311 extern char cml_version_number[];
315 * The following signal action for AIX is necessary so that in case of a
316 * crash (i.e. core is generated) we can include the user's data section
317 * in the core dump. Unfortunately, by default, only a partial core is
318 * generated which, in many cases, isn't too useful.
320 struct sigaction nsa;
322 sigemptyset(&nsa.sa_mask);
323 nsa.sa_handler = SIG_DFL;
324 nsa.sa_flags = SA_FULLDUMP;
325 sigaction(SIGABRT, &nsa, NULL);
326 sigaction(SIGSEGV, &nsa, NULL);
329 /* Initialize directory paths */
330 if (!(initAFSDirPath() & AFSDIR_SERVER_PATHS_OK)) {
332 ReportErrorEventAlt(AFSEVT_SVR_NO_INSTALL_DIR, 0, argv[0], 0);
334 fprintf(stderr, "%s: Unable to obtain AFS server directory.\n",
339 main_thread = pthread_self();
340 if (spawnDatap && spawnDataLen) {
341 /* This is a child per partition salvager. Don't setup log or
342 * try to lock the salvager lock.
344 if (nt_SetupPartitionSalvage(spawnDatap, spawnDataLen) < 0)
348 for (commandLine[0] = '\0', i = 0; i < argc; i++) {
350 strlcat(commandLine, " ", sizeof(commandLine));
351 strlcat(commandLine, argv[i], sizeof(commandLine));
355 if (geteuid() != 0) {
356 printf("Salvager must be run as root.\n");
362 /* bad for normal help flag processing, but can do nada */
368 ts = cmd_CreateSyntax("initcmd", handleit, 0, "initialize the program");
369 cmd_AddParm(ts, "-partition", CMD_SINGLE, CMD_OPTIONAL,
370 "Name of partition to salvage");
371 cmd_AddParm(ts, "-volumeid", CMD_SINGLE, CMD_OPTIONAL,
372 "Volume Id to salvage");
373 cmd_AddParm(ts, "-debug", CMD_FLAG, CMD_OPTIONAL,
374 "Run in Debugging mode");
375 cmd_AddParm(ts, "-nowrite", CMD_FLAG, CMD_OPTIONAL,
376 "Run readonly/test mode");
377 cmd_AddParm(ts, "-inodes", CMD_FLAG, CMD_OPTIONAL,
378 "Just list affected afs inodes - debugging flag");
379 cmd_AddParm(ts, "-oktozap", CMD_FLAG, CMD_OPTIONAL,
380 "Give permission to destroy bogus inodes/volumes - debugging flag");
381 cmd_AddParm(ts, "-rootinodes", CMD_FLAG, CMD_OPTIONAL,
382 "Show inodes owned by root - debugging flag");
383 cmd_AddParm(ts, "-salvagedirs", CMD_FLAG, CMD_OPTIONAL,
384 "Force rebuild/salvage of all directories");
385 cmd_AddParm(ts, "-blockreads", CMD_FLAG, CMD_OPTIONAL,
386 "Read smaller blocks to handle IO/bad blocks");
387 cmd_AddParm(ts, "-parallel", CMD_SINGLE, CMD_OPTIONAL,
388 "# of max parallel partition salvaging");
389 cmd_AddParm(ts, "-tmpdir", CMD_SINGLE, CMD_OPTIONAL,
390 "Name of dir to place tmp files ");
391 cmd_AddParm(ts, "-showlog", CMD_FLAG, CMD_OPTIONAL,
392 "Show log file upon completion");
393 cmd_AddParm(ts, "-orphans", CMD_SINGLE, CMD_OPTIONAL,
394 "ignore | remove | attach");
396 /* note - syslog isn't avail on NT, but if we make it conditional, have
397 * to deal with screwy offsets for cmd params */
398 cmd_AddParm(ts, "-syslog", CMD_FLAG, CMD_OPTIONAL,
399 "Write salvage log to syslogs");
400 cmd_AddParm(ts, "-syslogfacility", CMD_SINGLE, CMD_OPTIONAL,
401 "Syslog facility number to use");
402 cmd_AddParm(ts, "-datelogs", CMD_FLAG, CMD_OPTIONAL,
403 "Include timestamp in logfile filename");
405 cmd_AddParm(ts, "-client", CMD_FLAG, CMD_OPTIONAL,
406 "Use SALVSYNC to ask salvageserver to salvage a volume");
408 err = cmd_Dispatch(argc, argv);
413 SalvageClient(VolumeId vid, char * pname)
418 SALVSYNC_response_hdr sres;
420 VInitVolumePackage(volumeUtility, 5, 5, DONT_CONNECT_FS, 0);
421 SALVSYNC_clientInit();
423 code = SALVSYNC_SalvageVolume(vid, pname, SALVSYNC_SALVAGE, SALVSYNC_OPERATOR, 0, NULL);
424 if (code != SYNC_OK) {
428 res.payload.buf = (void *) &sres;
429 res.payload.len = sizeof(sres);
433 code = SALVSYNC_SalvageVolume(vid, pname, SALVSYNC_QUERY, SALVSYNC_WHATEVER, 0, &res);
434 if (code != SYNC_OK) {
437 switch (sres.state) {
438 case SALVSYNC_STATE_ERROR:
439 printf("salvageserver reports salvage ended in an error; check log files for more details\n");
440 case SALVSYNC_STATE_DONE:
441 case SALVSYNC_STATE_UNKNOWN:
445 SALVSYNC_clientFinis();
449 if (code == SYNC_DENIED) {
450 printf("salvageserver refused to salvage volume %u on partition %s\n",
452 } else if (code == SYNC_BAD_COMMAND) {
453 printf("SALVSYNC protocol mismatch; please make sure fileserver, volserver, salvageserver and salvager are same version\n");
454 } else if (code == SYNC_COM_ERROR) {
455 printf("SALVSYNC communications error\n");
457 SALVSYNC_clientFinis();
461 static int * child_slot;
467 struct SalvageQueueNode * node;
469 pthread_attr_t attrs;
472 /* All entries to the log will be appended. Useful if there are
473 * multiple salvagers appending to the log.
476 CheckLogFile(AFSDIR_SERVER_SALSRVLOG_FILEPATH);
478 #ifdef AFS_LINUX20_ENV
479 fcntl(fileno(logFile), F_SETFL, O_APPEND); /* Isn't this redundant? */
481 fcntl(fileno(logFile), F_SETFL, FAPPEND); /* Isn't this redundant? */
486 fprintf(logFile, "%s\n", cml_version_number);
487 Log("Starting OpenAFS Online Salvage Server %s (%s)\n", SalvageVersion, commandLine);
489 /* Get and hold a lock for the duration of the salvage to make sure
490 * that no other salvage runs at the same time. The routine
491 * VInitVolumePackage (called below) makes sure that a file server or
492 * other volume utilities don't interfere with the salvage.
495 /* even demand attach online salvager
496 * still needs this because we don't want
497 * a stand-alone salvager to conflict with
498 * the salvager daemon */
501 child_slot = (int *) malloc(Parallel * sizeof(int));
502 assert(child_slot != NULL);
503 memset(child_slot, 0, Parallel * sizeof(int));
505 /* initialize things */
506 VInitVolumePackage(salvageServer, 5, 5,
509 queue_Init(&pending_q);
510 queue_Init(&log_cleanup_queue);
511 assert(pthread_mutex_init(&worker_lock, NULL) == 0);
512 assert(pthread_cond_init(&worker_cv, NULL) == 0);
513 assert(pthread_cond_init(&log_cleanup_queue.queue_change_cv, NULL) == 0);
514 assert(pthread_attr_init(&attrs) == 0);
516 /* start up the reaper and log cleaner threads */
517 assert(pthread_attr_setdetachstate(&attrs, PTHREAD_CREATE_DETACHED) == 0);
518 assert(pthread_create(&tid,
520 &SalvageChildReaperThread,
522 assert(pthread_create(&tid,
524 &SalvageLogCleanupThread,
527 /* loop forever serving requests */
529 node = SALVSYNC_getWork();
530 assert(node != NULL);
534 for (slot = 0; slot < Parallel; slot++) {
535 if (!child_slot[slot])
538 assert (slot < Parallel);
543 ret = DoSalvageVolume(node, slot);
545 } else if (pid < 0) {
547 SALVSYNC_doneWork(node, 1);
549 child_slot[slot] = pid;
553 assert(pthread_mutex_lock(&worker_lock) == 0);
556 /* let the reaper thread know another worker was spawned */
557 assert(pthread_cond_broadcast(&worker_cv) == 0);
559 /* if we're overquota, wait for the reaper */
560 while (current_workers >= Parallel) {
561 assert(pthread_cond_wait(&worker_cv, &worker_lock) == 0);
563 assert(pthread_mutex_unlock(&worker_lock) == 0);
569 DoSalvageVolume(struct SalvageQueueNode * node, int slot)
571 char childLog[AFSDIR_PATH_MAX];
573 struct DiskPartition * partP;
575 VChildProcReconnectFS();
577 /* do not attempt to close parent's logFile handle as
578 * another thread may have held the lock on the FILE
579 * structure when fork was called! */
581 afs_snprintf(childLog, sizeof(childLog), "%s.%d",
582 AFSDIR_SERVER_SLVGLOG_FILEPATH, getpid());
584 logFile = afs_fopen(childLog, "a");
585 if (!logFile) { /* still nothing, use stdout */
590 if (node->command.sop.volume <= 0) {
591 Log("salvageServer: invalid volume id specified; salvage aborted\n");
595 partP = VGetPartition(node->command.sop.partName, 0);
597 Log("salvageServer: Unknown or unmounted partition %s; salvage aborted\n",
598 node->command.sop.partName);
602 /* Salvage individual volume; don't notify fs */
603 SalvageFileSys1(partP, node->command.sop.volume);
613 SalvageChildReaperThread(void * args)
615 int slot, pid, status, code, found;
616 struct SalvageQueueNode *qp, *nqp;
617 struct log_cleanup_node * cleanup;
619 assert(pthread_mutex_lock(&worker_lock) == 0);
621 /* loop reaping our children */
623 /* wait() won't block unless we have children, so
624 * block on the cond var if we're childless */
625 while (current_workers == 0) {
626 assert(pthread_cond_wait(&worker_cv, &worker_lock) == 0);
629 assert(pthread_mutex_unlock(&worker_lock) == 0);
631 cleanup = (struct log_cleanup_node *) malloc(sizeof(struct log_cleanup_node));
633 while (Reap_Child("salvageserver", &pid, &status) < 0) {
634 /* try to prevent livelock if something goes wrong */
639 for (slot = 0; slot < Parallel; slot++) {
640 if (child_slot[slot] == pid)
643 assert(slot < Parallel);
644 child_slot[slot] = 0;
647 assert(pthread_mutex_lock(&worker_lock) == 0);
651 queue_Append(&log_cleanup_queue, cleanup);
652 assert(pthread_cond_signal(&log_cleanup_queue.queue_change_cv) == 0);
655 /* ok, we've reaped a child */
657 SALVSYNC_doneWorkByPid(pid, 0);
658 assert(pthread_cond_broadcast(&worker_cv) == 0);
665 Reap_Child(char *prog, int * pid, int * status)
672 if (WCOREDUMP(*status))
673 Log("\"%s\" core dumped!\n", prog);
674 if (WIFSIGNALED(*status) != 0 || WEXITSTATUS(*status) != 0)
675 Log("\"%s\" (pid=%d) terminated abnormally!\n", prog, ret);
677 Log("wait returned -1\n");
683 * thread to combine salvager child logs
684 * back into the main salvageserver log
687 SalvageLogCleanupThread(void * arg)
689 struct log_cleanup_node * cleanup;
691 assert(pthread_mutex_lock(&worker_lock) == 0);
694 while (queue_IsEmpty(&log_cleanup_queue)) {
695 assert(pthread_cond_wait(&log_cleanup_queue.queue_change_cv, &worker_lock) == 0);
698 while (queue_IsNotEmpty(&log_cleanup_queue)) {
699 cleanup = queue_First(&log_cleanup_queue, log_cleanup_node);
700 queue_Remove(cleanup);
701 assert(pthread_mutex_unlock(&worker_lock) == 0);
702 SalvageLogCleanup(cleanup->pid);
704 assert(pthread_mutex_lock(&worker_lock) == 0);
708 assert(pthread_mutex_unlock(&worker_lock) == 0);
712 #define LOG_XFER_BUF_SIZE 65536
714 SalvageLogCleanup(int pid)
717 char fn[AFSDIR_PATH_MAX];
718 static char buf[LOG_XFER_BUF_SIZE];
720 afs_snprintf(fn, sizeof(fn), "%s.%d",
721 AFSDIR_SERVER_SLVGLOG_FILEPATH, pid);
724 pidlog = open(fn, O_RDONLY);
729 len = read(pidlog, buf, LOG_XFER_BUF_SIZE);
731 fwrite(buf, len, 1, logFile);
732 len = read(pidlog, buf, LOG_XFER_BUF_SIZE);