2 * Copyright 2000, International Business Machines Corporation and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
13 Institution: The Information Technology Center, Carnegie-Mellon University
18 /* All this is going away in early 1989 */
19 int newVLDB; /* Compatibility flag */
22 static int newVLDB = 1;
24 #ifndef AFS_PTHREAD_ENV
25 #define USUAL_PRIORITY (LWP_MAX_PRIORITY - 2)
28 * stack size increased from 8K because the HP machine seemed to have trouble
29 * with the smaller stack
31 #define USUAL_STACK_SIZE (24 * 1024)
32 #endif /* !AFS_PTHREAD_ENV */
36 File server synchronization with external volume utilities.
38 #include <afsconfig.h>
39 #include <afs/param.h>
44 #include <sys/types.h>
50 #include <sys/param.h>
51 #include <sys/socket.h>
52 #include <netinet/in.h>
57 #ifdef AFS_PTHREAD_ENV
59 #else /* AFS_PTHREAD_ENV */
60 #include <afs/assert.h>
61 #endif /* AFS_PTHREAD_ENV */
74 #include <afs/afsint.h>
76 #include <afs/errors.h>
80 #include <afs/afssyscalls.h>
84 #include "partition.h"
86 /*@printflike@*/ extern void Log(const char *format, ...);
91 #define osi_Assert(e) (void)(e)
93 extern int LogLevel; /* Vice loglevel */
94 int (*V_BreakVolumeCallbacks) ();
96 #define MAXHANDLERS 4 /* Up to 4 clients; must be at least 2, so that
97 * move = dump+restore can run on single server */
98 #define MAXOFFLINEVOLUMES 30 /* This needs to be as big as the maximum
99 * number that would be offline for 1 operation.
100 * Current winner is salvage, which needs all
101 * cloned read-only copies offline when salvaging
102 * a single read-write volume */
104 #define MAX_BIND_TRIES 5 /* Number of times to retry socket bind */
112 static struct offlineInfo OfflineVolumes[MAXHANDLERS][MAXOFFLINEVOLUMES];
114 static FS_sd = -1; /* Client socket for talking to file server */
115 static AcceptSd = -1; /* Socket used by server for accepting connections */
117 static int getport();
123 char partName[16]; /* partition name, e.g. /vicepa */
127 /* Forward declarations */
128 static int getport();
129 static void FSYNC_sync();
130 static void FSYNC_newconnection();
131 static void FSYNC_com();
132 static void FSYNC_Drop();
133 static void AcceptOn();
134 static void AcceptOff();
135 static void InitHandler();
136 static void CallHandler(fd_set * fdsetp);
137 static int AddHandler();
138 static int FindHandler();
139 static int FindHandler_r();
140 static int RemoveHandler();
141 static void GetHandler(fd_set * fdsetp, int *maxfdp);
144 * This lock controls access to the handler array. The overhead
145 * is minimal in non-preemptive environments.
147 struct Lock FSYNC_handler_lock;
150 FSYNC_clientInit(void)
152 struct sockaddr_in addr;
153 /* I can't believe the following is needed for localhost connections!! */
154 static time_t backoff[] =
155 { 3, 3, 3, 5, 5, 5, 7, 15, 16, 24, 32, 40, 48, 0 };
156 time_t *timeout = &backoff[0];
159 FS_sd = getport(&addr);
160 if (connect(FS_sd, (struct sockaddr *)&addr, sizeof(addr)) >= 0)
162 #if defined(AFS_SGI_ENV)
163 /* down with worthless error messages! */
165 perror("FSYNC_clientInit failed (after many retries)");
172 perror("FSYNC_clientInit temporary failure (will retry)");
177 perror("FSYNC_clientInit failed (giving up!)");
182 FSYNC_clientFinis(void)
190 Lock_Destroy(&FSYNC_handler_lock);
194 FSYNC_askfs(VolumeId volume, char *partName, int com, int reason)
197 struct command command;
199 command.volume = volume;
200 command.command = com;
201 command.reason = reason;
203 strcpy(command.partName, partName);
205 command.partName[0] = 0;
208 if (send(FS_sd, (char *)&command, sizeof(command), 0) != sizeof(command)) {
209 printf("FSYNC_askfs: write to file server failed\n");
212 while ((n = recv(FS_sd, &response, 1, 0)) != 1) {
213 if (n == 0 || WSAEINTR != WSAGetLastError()) {
214 printf("FSYNC_askfs: No response from file server\n");
219 if (write(FS_sd, &command, sizeof(command)) != sizeof(command)) {
220 printf("FSYNC_askfs: write to file server failed\n");
223 while ((n = read(FS_sd, &response, 1)) != 1) {
224 if (n == 0 || errno != EINTR) {
225 printf("FSYNC_askfs: No response from file server\n");
232 ("FSYNC_askfs: negative response from file server; volume %u, command %d\n",
233 command.volume, (int)command.command);
242 #ifdef AFS_PTHREAD_ENV
244 pthread_attr_t tattr;
245 assert(pthread_attr_init(&tattr) == 0);
246 assert(pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED) == 0);
247 assert(pthread_create(&tid, &tattr, FSYNC_sync, NULL) == 0);
248 #else /* AFS_PTHREAD_ENV */
250 assert(LWP_CreateProcess
251 (FSYNC_sync, USUAL_STACK_SIZE, USUAL_PRIORITY, (void *)0,
252 "FSYNC_sync", &pid) == LWP_SUCCESS);
253 #endif /* AFS_PTHREAD_ENV */
257 getport(struct sockaddr_in *addr)
261 memset(addr, 0, sizeof(*addr));
262 assert((sd = socket(AF_INET, SOCK_STREAM, 0)) >= 0);
263 #ifdef STRUCT_SOCKADDR_HAS_SA_LEN
264 addr->sin_len = sizeof(struct sockaddr_in);
266 addr->sin_addr.s_addr = htonl(0x7f000001);
267 addr->sin_family = AF_INET; /* was localhost->h_addrtype */
268 addr->sin_port = htons(2040); /* XXXX htons not _really_ neccessary */
276 struct sockaddr_in addr;
281 #ifdef AFS_PTHREAD_ENV
286 (void)signal(SIGPIPE, SIG_IGN);
289 #ifdef AFS_PTHREAD_ENV
290 /* set our 'thread-id' so that the host hold table works */
291 MUTEX_ENTER(&rx_stats_mutex); /* protects rxi_pthread_hinum */
292 tid = ++rxi_pthread_hinum;
293 MUTEX_EXIT(&rx_stats_mutex);
294 pthread_setspecific(rx_thread_id_key, (void *)tid);
295 Log("Set thread id %d for FSYNC_sync\n", tid);
296 #endif /* AFS_PTHREAD_ENV */
299 /* Let somebody else run until level > 0. That doesn't mean that
300 * all volumes have been attached. */
301 #ifdef AFS_PTHREAD_ENV
303 #else /* AFS_PTHREAD_ENV */
304 LWP_DispatchProcess();
305 #endif /* AFS_PTHREAD_ENV */
307 AcceptSd = getport(&addr);
308 /* Reuseaddr needed because system inexplicably leaves crud lying around */
310 setsockopt(AcceptSd, SOL_SOCKET, SO_REUSEADDR, (char *)&on,
313 Log("FSYNC_sync: setsockopt failed with (%d)\n", errno);
315 for (numTries = 0; numTries < MAX_BIND_TRIES; numTries++) {
317 bind(AcceptSd, (struct sockaddr *)&addr, sizeof(addr))) == 0)
319 Log("FSYNC_sync: bind failed with (%d), will sleep and retry\n",
324 listen(AcceptSd, 100);
330 GetHandler(&readfds, &maxfd);
331 /* Note: check for >= 1 below is essential since IOMGR_select
332 * doesn't have exactly same semantics as select.
334 #ifdef AFS_PTHREAD_ENV
335 if (select(maxfd + 1, &readfds, NULL, NULL, NULL) >= 1)
336 #else /* AFS_PTHREAD_ENV */
337 if (IOMGR_Select(maxfd + 1, &readfds, NULL, NULL, NULL) >= 1)
338 #endif /* AFS_PTHREAD_ENV */
339 CallHandler(&readfds);
344 FSYNC_newconnection(int afd)
346 struct sockaddr_in other;
348 junk = sizeof(other);
349 fd = accept(afd, (struct sockaddr *)&other, &junk);
351 Log("FSYNC_newconnection: accept failed, errno==%d\n", errno);
353 } else if (!AddHandler(fd, FSYNC_com)) {
355 assert(AddHandler(fd, FSYNC_com));
363 afs_int32 FS_cnt = 0;
370 struct command command;
372 register struct offlineInfo *volumes, *v;
374 char tvolName[VMAXPATHLEN];
378 n = recv(fd, &command, sizeof(command), 0);
380 n = read(fd, &command, sizeof(command));
386 if (n < sizeof(command)) {
387 Log("FSYNC_com: partial read (%d instead of %d); dropping connection (cnt=%d)\n", n, sizeof(command), FS_cnt);
391 VATTACH_LOCK VOL_LOCK volumes = OfflineVolumes[FindHandler(fd)];
392 for (v = 0, i = 0; i < MAXOFFLINEVOLUMES; i++) {
393 if (volumes[i].volumeID == command.volume
394 && strcmp(volumes[i].partName, command.partName) == 0) {
399 switch (command.command) {
401 /* don't try to put online, this call is made only after deleting
402 * a volume, in which case we want to remove the vol # from the
403 * OfflineVolumes array only */
410 This is where a detatched volume gets reattached. However in the
411 special case where the volume is merely busy, it is already
412 attatched and it is only necessary to clear the busy flag. See
413 defect #2080 for details.
416 /* is the volume already attatched? */
419 * XXX With the following enabled we had bizarre problems where the backup id would
420 * be reset to 0; that was due to the interaction between fileserver/volserver in that they
421 * both keep volumes in memory and the changes wouldn't be made to the fileserver. Some of
422 * the problems were due to refcnt changes as result of VGetVolume/VPutVolume which would call
423 * VOffline, etc. when we don't want to; someday the whole #2080 issue should be revisited to
426 vp = VGetVolume_r(&error, command.volume);
428 /* yep, is the BUSY flag set? */
429 if (vp->specialStatus == VBUSY) {
430 /* test harness for defect #2081 */
434 * test #2081 by releasing TEST.2081,
435 * so leave it alone here, zap it after
438 if (strcmp(vp->header->diskstuff.name, "TEST.2081") == 0)
441 /* yep, clear BUSY flag */
443 vp->specialStatus = 0;
444 /* make sure vol is online */
447 V_inUse(vp) = 1; /* online */
456 /* so, we need to attach the volume */
461 sprintf(&tvolName[1], VFORMAT, command.volume);
463 vp = VAttachVolumeByName_r(&error, command.partName, tvolName,
469 case FSYNC_NEEDVOLUME:{
471 /* not already offline, we need to find a slot for newly offline volume */
473 for (i = 0; i < MAXOFFLINEVOLUMES; i++) {
474 if (volumes[i].volumeID == 0) {
484 vp = VGetVolume_r(&error, command.volume);
486 if (command.partName[0] != 0
487 && strcmp(command.partName, vp->partition->name) != 0) {
488 /* volume on desired partition is not online, so we
489 * should treat this as an offline volume.
496 leaveonline = (command.command == FSYNC_NEEDVOLUME
497 && (command.reason == V_READONLY
498 || (!VolumeWriteable(vp)
499 && (command.reason == V_CLONE
500 || command.reason == V_DUMP))
504 if (command.command == FSYNC_NEEDVOLUME
505 && (command.reason == V_CLONE
506 || command.reason == V_DUMP)) {
507 vp->specialStatus = VBUSY;
509 /* remember what volume we got, so we can keep track of how
510 * many volumes the volserver or whatever is using. Note that
511 * vp is valid since leaveonline is only set when vp is valid.
513 v->volumeID = command.volume;
514 strcpy(v->partName, vp->partition->name);
516 /* in this case, VOffline just returns sans decrementing
517 * ref count. We could try to fix it, but it has lots of
522 VOffline_r(vp, "A volume utility is running.");
526 VUpdateVolume_r(&error, vp); /* At least get volume stats right */
528 Log("FSYNC: Volume %u (%s) was left on line for an external %s request\n", V_id(vp), V_name(vp), command.reason == V_CLONE ? "clone" : command.reason == V_READONLY ? "readonly" : command.reason == V_DUMP ? "dump" : "UNKNOWN");
537 case FSYNC_MOVEVOLUME:
538 /* Yuch: the "reason" for the move is the site it got moved to... */
539 /* still set specialStatus so we stop sending back VBUSY.
540 * also should still break callbacks. Note that I don't know
541 * how to tell if we should break all or not, so we just do it
542 * since it doesn't matter much if we do an extra break
543 * volume callbacks on a volume move within the same server */
544 vp = VGetVolume_r(&error, command.volume);
546 vp->specialStatus = VMOVED;
550 if (V_BreakVolumeCallbacks) {
551 Log("fssync: volume %u moved to %x; breaking all call backs\n",
552 command.volume, command.reason);
553 VOL_UNLOCK VATTACH_UNLOCK(*V_BreakVolumeCallbacks) (command.
555 VATTACH_LOCK VOL_LOCK}
557 case FSYNC_RESTOREVOLUME:
558 /* if the volume is being restored, break all callbacks on it */
559 if (V_BreakVolumeCallbacks) {
560 Log("fssync: volume %u restored; breaking all call backs\n",
562 VOL_UNLOCK VATTACH_UNLOCK(*V_BreakVolumeCallbacks) (command.
564 VATTACH_LOCK VOL_LOCK}
570 VOL_UNLOCK VATTACH_UNLOCK
572 (void) send(fd, &rc, 1, 0);
574 (void) write(fd, &rc, 1);
581 struct offlineInfo *p;
584 char tvolName[VMAXPATHLEN];
586 VATTACH_LOCK VOL_LOCK p = OfflineVolumes[FindHandler(fd)];
587 for (i = 0; i < MAXOFFLINEVOLUMES; i++) {
592 sprintf(&tvolName[1], VFORMAT, p[i].volumeID);
593 vp = VAttachVolumeByName_r(&error, p[i].partName, tvolName,
600 VOL_UNLOCK VATTACH_UNLOCK RemoveHandler(fd);
609 static int AcceptHandler = -1; /* handler id for accept, if turned on */
614 if (AcceptHandler == -1) {
615 assert(AddHandler(AcceptSd, FSYNC_newconnection));
616 AcceptHandler = FindHandler(AcceptSd);
623 if (AcceptHandler != -1) {
624 assert(RemoveHandler(AcceptSd));
629 /* The multiple FD handling code. */
631 static int HandlerFD[MAXHANDLERS];
632 static int (*HandlerProc[MAXHANDLERS]) ();
638 ObtainWriteLock(&FSYNC_handler_lock);
639 for (i = 0; i < MAXHANDLERS; i++) {
643 ReleaseWriteLock(&FSYNC_handler_lock);
647 CallHandler(fd_set * fdsetp)
650 ObtainReadLock(&FSYNC_handler_lock);
651 for (i = 0; i < MAXHANDLERS; i++) {
652 if (HandlerFD[i] >= 0 && FD_ISSET(HandlerFD[i], fdsetp)) {
653 ReleaseReadLock(&FSYNC_handler_lock);
654 (*HandlerProc[i]) (HandlerFD[i]);
655 ObtainReadLock(&FSYNC_handler_lock);
658 ReleaseReadLock(&FSYNC_handler_lock);
662 AddHandler(int afd, int (*aproc) ())
665 ObtainWriteLock(&FSYNC_handler_lock);
666 for (i = 0; i < MAXHANDLERS; i++)
667 if (HandlerFD[i] == -1)
669 if (i >= MAXHANDLERS) {
670 ReleaseWriteLock(&FSYNC_handler_lock);
674 HandlerProc[i] = aproc;
675 ReleaseWriteLock(&FSYNC_handler_lock);
680 FindHandler(register int afd)
683 ObtainReadLock(&FSYNC_handler_lock);
684 for (i = 0; i < MAXHANDLERS; i++)
685 if (HandlerFD[i] == afd) {
686 ReleaseReadLock(&FSYNC_handler_lock);
689 ReleaseReadLock(&FSYNC_handler_lock); /* just in case */
691 return -1; /* satisfy compiler */
695 FindHandler_r(register int afd)
698 for (i = 0; i < MAXHANDLERS; i++)
699 if (HandlerFD[i] == afd) {
703 return -1; /* satisfy compiler */
707 RemoveHandler(register int afd)
709 ObtainWriteLock(&FSYNC_handler_lock);
710 HandlerFD[FindHandler_r(afd)] = -1;
711 ReleaseWriteLock(&FSYNC_handler_lock);
716 GetHandler(fd_set * fdsetp, int *maxfdp)
719 register int maxfd = -1;
721 ObtainReadLock(&FSYNC_handler_lock); /* just in case */
722 for (i = 0; i < MAXHANDLERS; i++)
723 if (HandlerFD[i] != -1) {
724 FD_SET(HandlerFD[i], fdsetp);
725 if (maxfd < HandlerFD[i])
726 maxfd = HandlerFD[i];
729 ReleaseReadLock(&FSYNC_handler_lock); /* just in case */