2 * Copyright 2000, International Business Machines Corporation and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
13 Institution: The Information Technology Center, Carnegie-Mellon University
18 /* All this is going away in early 1989 */
19 int newVLDB; /* Compatibility flag */
22 static int newVLDB = 1;
24 #ifndef AFS_PTHREAD_ENV
25 #define USUAL_PRIORITY (LWP_MAX_PRIORITY - 2)
28 * stack size increased from 8K because the HP machine seemed to have trouble
29 * with the smaller stack
31 #define USUAL_STACK_SIZE (24 * 1024)
32 #endif /* !AFS_PTHREAD_ENV */
36 File server synchronization with external volume utilities.
38 #include <afsconfig.h>
39 #include <afs/param.h>
44 #include <sys/types.h>
50 #include <sys/param.h>
51 #include <sys/socket.h>
52 #include <netinet/in.h>
57 #ifdef AFS_PTHREAD_ENV
59 #else /* AFS_PTHREAD_ENV */
60 #include <afs/assert.h>
61 #endif /* AFS_PTHREAD_ENV */
74 #include <afs/afsint.h>
76 #include <afs/errors.h>
80 #include <afs/afssyscalls.h>
84 #include "partition.h"
86 /*@printflike@*/ extern void Log(const char *format, ...);
91 #define osi_Assert(e) (void)(e)
93 extern int LogLevel; /* Vice loglevel */
94 int (*V_BreakVolumeCallbacks) ();
96 #define MAXHANDLERS 4 /* Up to 4 clients; must be at least 2, so that
97 * move = dump+restore can run on single server */
98 #define MAXOFFLINEVOLUMES 30 /* This needs to be as big as the maximum
99 * number that would be offline for 1 operation.
100 * Current winner is salvage, which needs all
101 * cloned read-only copies offline when salvaging
102 * a single read-write volume */
104 #define MAX_BIND_TRIES 5 /* Number of times to retry socket bind */
112 static struct offlineInfo OfflineVolumes[MAXHANDLERS][MAXOFFLINEVOLUMES];
114 static FS_sd = -1; /* Client socket for talking to file server */
115 static AcceptSd = -1; /* Socket used by server for accepting connections */
117 static int getport();
123 char partName[16]; /* partition name, e.g. /vicepa */
127 /* Forward declarations */
128 static int getport();
129 static void FSYNC_sync();
130 static void FSYNC_newconnection();
131 static void FSYNC_com();
132 static void FSYNC_Drop();
133 static void AcceptOn();
134 static void AcceptOff();
135 static void InitHandler();
136 static void CallHandler(fd_set * fdsetp);
137 static int AddHandler();
138 static int FindHandler();
139 static int FindHandler_r();
140 static int RemoveHandler();
141 static void GetHandler(fd_set * fdsetp, int *maxfdp);
144 * This lock controls access to the handler array. The overhead
145 * is minimal in non-preemptive environments.
147 struct Lock FSYNC_handler_lock;
150 FSYNC_clientInit(void)
152 struct sockaddr_in addr;
153 /* I can't believe the following is needed for localhost connections!! */
154 static time_t backoff[] =
155 { 3, 3, 3, 5, 5, 5, 7, 15, 16, 24, 32, 40, 48, 0 };
156 time_t *timeout = &backoff[0];
159 FS_sd = getport(&addr);
160 if (connect(FS_sd, (struct sockaddr *)&addr, sizeof(addr)) >= 0)
162 #if defined(AFS_SGI_ENV)
163 /* down with worthless error messages! */
165 perror("FSYNC_clientInit failed (after many retries)");
172 perror("FSYNC_clientInit temporary failure (will retry)");
177 perror("FSYNC_clientInit failed (giving up!)");
182 FSYNC_clientFinis(void)
190 Lock_Destroy(&FSYNC_handler_lock);
194 FSYNC_askfs(VolumeId volume, char *partName, int com, int reason)
197 struct command command;
199 command.volume = volume;
200 command.command = com;
201 command.reason = reason;
203 strcpy(command.partName, partName);
205 command.partName[0] = 0;
209 if (send(FS_sd, (char *)&command, sizeof(command), 0) != sizeof(command)) {
210 printf("FSYNC_askfs: write to file server failed\n");
211 response = FSYNC_DENIED;
214 while ((n = recv(FS_sd, &response, 1, 0)) != 1) {
215 if (n == 0 || WSAEINTR != WSAGetLastError()) {
216 printf("FSYNC_askfs: No response from file server\n");
217 response = FSYNC_DENIED;
222 if (write(FS_sd, &command, sizeof(command)) != sizeof(command)) {
223 printf("FSYNC_askfs: write to file server failed\n");
224 response = FSYNC_DENIED;
227 while ((n = read(FS_sd, &response, 1)) != 1) {
228 if (n == 0 || errno != EINTR) {
229 printf("FSYNC_askfs: No response from file server\n");
230 response = FSYNC_DENIED;
237 ("FSYNC_askfs: negative response from file server; volume %u, command %d\n",
238 command.volume, (int)command.command);
249 #ifdef AFS_PTHREAD_ENV
251 pthread_attr_t tattr;
252 assert(pthread_attr_init(&tattr) == 0);
253 assert(pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED) == 0);
254 assert(pthread_create(&tid, &tattr, FSYNC_sync, NULL) == 0);
255 #else /* AFS_PTHREAD_ENV */
257 assert(LWP_CreateProcess
258 (FSYNC_sync, USUAL_STACK_SIZE, USUAL_PRIORITY, (void *)0,
259 "FSYNC_sync", &pid) == LWP_SUCCESS);
260 #endif /* AFS_PTHREAD_ENV */
264 getport(struct sockaddr_in *addr)
268 memset(addr, 0, sizeof(*addr));
269 assert((sd = socket(AF_INET, SOCK_STREAM, 0)) >= 0);
270 #ifdef STRUCT_SOCKADDR_HAS_SA_LEN
271 addr->sin_len = sizeof(struct sockaddr_in);
273 addr->sin_addr.s_addr = htonl(0x7f000001);
274 addr->sin_family = AF_INET; /* was localhost->h_addrtype */
275 addr->sin_port = htons(2040); /* XXXX htons not _really_ neccessary */
283 struct sockaddr_in addr;
288 #ifdef AFS_PTHREAD_ENV
293 (void)signal(SIGPIPE, SIG_IGN);
296 #ifdef AFS_PTHREAD_ENV
297 /* set our 'thread-id' so that the host hold table works */
298 MUTEX_ENTER(&rx_stats_mutex); /* protects rxi_pthread_hinum */
299 tid = ++rxi_pthread_hinum;
300 MUTEX_EXIT(&rx_stats_mutex);
301 pthread_setspecific(rx_thread_id_key, (void *)tid);
302 Log("Set thread id %d for FSYNC_sync\n", tid);
303 #endif /* AFS_PTHREAD_ENV */
306 /* Let somebody else run until level > 0. That doesn't mean that
307 * all volumes have been attached. */
308 #ifdef AFS_PTHREAD_ENV
310 #else /* AFS_PTHREAD_ENV */
311 LWP_DispatchProcess();
312 #endif /* AFS_PTHREAD_ENV */
314 AcceptSd = getport(&addr);
315 /* Reuseaddr needed because system inexplicably leaves crud lying around */
317 setsockopt(AcceptSd, SOL_SOCKET, SO_REUSEADDR, (char *)&on,
320 Log("FSYNC_sync: setsockopt failed with (%d)\n", errno);
322 for (numTries = 0; numTries < MAX_BIND_TRIES; numTries++) {
324 bind(AcceptSd, (struct sockaddr *)&addr, sizeof(addr))) == 0)
326 Log("FSYNC_sync: bind failed with (%d), will sleep and retry\n",
331 listen(AcceptSd, 100);
337 GetHandler(&readfds, &maxfd);
338 /* Note: check for >= 1 below is essential since IOMGR_select
339 * doesn't have exactly same semantics as select.
341 #ifdef AFS_PTHREAD_ENV
342 if (select(maxfd + 1, &readfds, NULL, NULL, NULL) >= 1)
343 #else /* AFS_PTHREAD_ENV */
344 if (IOMGR_Select(maxfd + 1, &readfds, NULL, NULL, NULL) >= 1)
345 #endif /* AFS_PTHREAD_ENV */
346 CallHandler(&readfds);
351 FSYNC_newconnection(int afd)
353 struct sockaddr_in other;
355 junk = sizeof(other);
356 fd = accept(afd, (struct sockaddr *)&other, &junk);
358 Log("FSYNC_newconnection: accept failed, errno==%d\n", errno);
360 } else if (!AddHandler(fd, FSYNC_com)) {
362 assert(AddHandler(fd, FSYNC_com));
370 afs_int32 FS_cnt = 0;
377 struct command command;
379 register struct offlineInfo *volumes, *v;
381 char tvolName[VMAXPATHLEN];
385 n = recv(fd, &command, sizeof(command), 0);
387 n = read(fd, &command, sizeof(command));
393 if (n < sizeof(command)) {
394 Log("FSYNC_com: partial read (%d instead of %d); dropping connection (cnt=%d)\n", n, sizeof(command), FS_cnt);
398 VATTACH_LOCK VOL_LOCK volumes = OfflineVolumes[FindHandler(fd)];
399 for (v = 0, i = 0; i < MAXOFFLINEVOLUMES; i++) {
400 if (volumes[i].volumeID == command.volume
401 && strcmp(volumes[i].partName, command.partName) == 0) {
406 switch (command.command) {
408 /* don't try to put online, this call is made only after deleting
409 * a volume, in which case we want to remove the vol # from the
410 * OfflineVolumes array only */
417 This is where a detatched volume gets reattached. However in the
418 special case where the volume is merely busy, it is already
419 attatched and it is only necessary to clear the busy flag. See
420 defect #2080 for details.
423 /* is the volume already attatched? */
426 * XXX With the following enabled we had bizarre problems where the backup id would
427 * be reset to 0; that was due to the interaction between fileserver/volserver in that they
428 * both keep volumes in memory and the changes wouldn't be made to the fileserver. Some of
429 * the problems were due to refcnt changes as result of VGetVolume/VPutVolume which would call
430 * VOffline, etc. when we don't want to; someday the whole #2080 issue should be revisited to
433 vp = VGetVolume_r(&error, command.volume);
435 /* yep, is the BUSY flag set? */
436 if (vp->specialStatus == VBUSY) {
437 /* test harness for defect #2081 */
441 * test #2081 by releasing TEST.2081,
442 * so leave it alone here, zap it after
445 if (strcmp(vp->header->diskstuff.name, "TEST.2081") == 0)
448 /* yep, clear BUSY flag */
450 vp->specialStatus = 0;
451 /* make sure vol is online */
454 V_inUse(vp) = 1; /* online */
463 /* so, we need to attach the volume */
468 sprintf(&tvolName[1], VFORMAT, command.volume);
470 vp = VAttachVolumeByName_r(&error, command.partName, tvolName,
476 case FSYNC_NEEDVOLUME:{
478 /* not already offline, we need to find a slot for newly offline volume */
480 for (i = 0; i < MAXOFFLINEVOLUMES; i++) {
481 if (volumes[i].volumeID == 0) {
491 vp = VGetVolume_r(&error, command.volume);
493 if (command.partName[0] != 0
494 && strcmp(command.partName, vp->partition->name) != 0) {
495 /* volume on desired partition is not online, so we
496 * should treat this as an offline volume.
503 leaveonline = (command.command == FSYNC_NEEDVOLUME
504 && (command.reason == V_READONLY
505 || (!VolumeWriteable(vp)
506 && (command.reason == V_CLONE
507 || command.reason == V_DUMP))
511 if (command.command == FSYNC_NEEDVOLUME
512 && (command.reason == V_CLONE
513 || command.reason == V_DUMP)) {
514 vp->specialStatus = VBUSY;
516 /* remember what volume we got, so we can keep track of how
517 * many volumes the volserver or whatever is using. Note that
518 * vp is valid since leaveonline is only set when vp is valid.
520 v->volumeID = command.volume;
521 strcpy(v->partName, vp->partition->name);
523 /* in this case, VOffline just returns sans decrementing
524 * ref count. We could try to fix it, but it has lots of
529 VOffline_r(vp, "A volume utility is running.");
533 VUpdateVolume_r(&error, vp); /* At least get volume stats right */
535 Log("FSYNC: Volume %u (%s) was left on line for an external %s request\n", V_id(vp), V_name(vp), command.reason == V_CLONE ? "clone" : command.reason == V_READONLY ? "readonly" : command.reason == V_DUMP ? "dump" : "UNKNOWN");
544 case FSYNC_MOVEVOLUME:
545 /* Yuch: the "reason" for the move is the site it got moved to... */
546 /* still set specialStatus so we stop sending back VBUSY.
547 * also should still break callbacks. Note that I don't know
548 * how to tell if we should break all or not, so we just do it
549 * since it doesn't matter much if we do an extra break
550 * volume callbacks on a volume move within the same server */
551 vp = VGetVolume_r(&error, command.volume);
553 vp->specialStatus = VMOVED;
557 if (V_BreakVolumeCallbacks) {
558 Log("fssync: volume %u moved to %x; breaking all call backs\n",
559 command.volume, command.reason);
560 VOL_UNLOCK VATTACH_UNLOCK(*V_BreakVolumeCallbacks) (command.
562 VATTACH_LOCK VOL_LOCK}
564 case FSYNC_RESTOREVOLUME:
565 /* if the volume is being restored, break all callbacks on it */
566 if (V_BreakVolumeCallbacks) {
567 Log("fssync: volume %u restored; breaking all call backs\n",
569 VOL_UNLOCK VATTACH_UNLOCK(*V_BreakVolumeCallbacks) (command.
571 VATTACH_LOCK VOL_LOCK}
577 VOL_UNLOCK VATTACH_UNLOCK
579 (void) send(fd, &rc, 1, 0);
581 (void) write(fd, &rc, 1);
588 struct offlineInfo *p;
591 char tvolName[VMAXPATHLEN];
593 VATTACH_LOCK VOL_LOCK p = OfflineVolumes[FindHandler(fd)];
594 for (i = 0; i < MAXOFFLINEVOLUMES; i++) {
599 sprintf(&tvolName[1], VFORMAT, p[i].volumeID);
600 vp = VAttachVolumeByName_r(&error, p[i].partName, tvolName,
607 VOL_UNLOCK VATTACH_UNLOCK RemoveHandler(fd);
616 static int AcceptHandler = -1; /* handler id for accept, if turned on */
621 if (AcceptHandler == -1) {
622 assert(AddHandler(AcceptSd, FSYNC_newconnection));
623 AcceptHandler = FindHandler(AcceptSd);
630 if (AcceptHandler != -1) {
631 assert(RemoveHandler(AcceptSd));
636 /* The multiple FD handling code. */
638 static int HandlerFD[MAXHANDLERS];
639 static int (*HandlerProc[MAXHANDLERS]) ();
645 ObtainWriteLock(&FSYNC_handler_lock);
646 for (i = 0; i < MAXHANDLERS; i++) {
650 ReleaseWriteLock(&FSYNC_handler_lock);
654 CallHandler(fd_set * fdsetp)
657 ObtainReadLock(&FSYNC_handler_lock);
658 for (i = 0; i < MAXHANDLERS; i++) {
659 if (HandlerFD[i] >= 0 && FD_ISSET(HandlerFD[i], fdsetp)) {
660 ReleaseReadLock(&FSYNC_handler_lock);
661 (*HandlerProc[i]) (HandlerFD[i]);
662 ObtainReadLock(&FSYNC_handler_lock);
665 ReleaseReadLock(&FSYNC_handler_lock);
669 AddHandler(int afd, int (*aproc) ())
672 ObtainWriteLock(&FSYNC_handler_lock);
673 for (i = 0; i < MAXHANDLERS; i++)
674 if (HandlerFD[i] == -1)
676 if (i >= MAXHANDLERS) {
677 ReleaseWriteLock(&FSYNC_handler_lock);
681 HandlerProc[i] = aproc;
682 ReleaseWriteLock(&FSYNC_handler_lock);
687 FindHandler(register int afd)
690 ObtainReadLock(&FSYNC_handler_lock);
691 for (i = 0; i < MAXHANDLERS; i++)
692 if (HandlerFD[i] == afd) {
693 ReleaseReadLock(&FSYNC_handler_lock);
696 ReleaseReadLock(&FSYNC_handler_lock); /* just in case */
698 return -1; /* satisfy compiler */
702 FindHandler_r(register int afd)
705 for (i = 0; i < MAXHANDLERS; i++)
706 if (HandlerFD[i] == afd) {
710 return -1; /* satisfy compiler */
714 RemoveHandler(register int afd)
716 ObtainWriteLock(&FSYNC_handler_lock);
717 HandlerFD[FindHandler_r(afd)] = -1;
718 ReleaseWriteLock(&FSYNC_handler_lock);
723 GetHandler(fd_set * fdsetp, int *maxfdp)
726 register int maxfd = -1;
728 ObtainReadLock(&FSYNC_handler_lock); /* just in case */
729 for (i = 0; i < MAXHANDLERS; i++)
730 if (HandlerFD[i] != -1) {
731 FD_SET(HandlerFD[i], fdsetp);
732 if (maxfd < HandlerFD[i])
733 maxfd = HandlerFD[i];
736 ReleaseReadLock(&FSYNC_handler_lock); /* just in case */