2 * Copyright 2000, International Business Machines Corporation and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
13 Institution: The Information Technology Center, Carnegie-Mellon University
18 /* All this is going away in early 1989 */
19 int newVLDB; /* Compatibility flag */
22 static int newVLDB = 1;
24 #ifndef AFS_PTHREAD_ENV
25 #define USUAL_PRIORITY (LWP_MAX_PRIORITY - 2)
28 * stack size increased from 8K because the HP machine seemed to have trouble
29 * with the smaller stack
31 #define USUAL_STACK_SIZE (24 * 1024)
32 #endif /* !AFS_PTHREAD_ENV */
36 File server synchronization with external volume utilities.
38 #include <afsconfig.h>
39 #include <afs/param.h>
43 #include <sys/types.h>
49 #include <sys/param.h>
50 #include <sys/socket.h>
51 #include <netinet/in.h>
56 #ifdef AFS_PTHREAD_ENV
58 #else /* AFS_PTHREAD_ENV */
59 #include <afs/assert.h>
60 #endif /* AFS_PTHREAD_ENV */
73 #include <afs/afsint.h>
75 #include <afs/errors.h>
79 #include <afs/afssyscalls.h>
83 #include "partition.h"
85 extern int LogLevel; /* Vice loglevel */
86 int (*V_BreakVolumeCallbacks)();
88 #define MAXHANDLERS 4 /* Up to 4 clients; must be at least 2, so that
89 move = dump+restore can run on single server */
90 #define MAXOFFLINEVOLUMES 30 /* This needs to be as big as the maximum
91 number that would be offline for 1 operation.
92 Current winner is salvage, which needs all
93 cloned read-only copies offline when salvaging
94 a single read-write volume */
96 #define MAX_BIND_TRIES 5 /* Number of times to retry socket bind */
104 static struct offlineInfo OfflineVolumes[MAXHANDLERS][MAXOFFLINEVOLUMES];
106 static FS_sd = -1; /* Client socket for talking to file server */
107 static AcceptSd = -1; /* Socket used by server for accepting connections */
109 static int getport();
115 char partName[16]; /* partition name, e.g. /vicepa */
119 /* Forward declarations */
120 static int getport();
121 static void FSYNC_sync();
122 static void FSYNC_newconnection();
123 static void FSYNC_com();
124 static void FSYNC_Drop();
125 static void AcceptOn();
126 static void AcceptOff();
127 static void InitHandler();
128 static void CallHandler(fd_set *fdsetp);
129 static int AddHandler();
130 static int FindHandler();
131 static int FindHandler_r();
132 static int RemoveHandler();
133 static void GetHandler(fd_set *fdsetp, int *maxfdp);
136 * This lock controls access to the handler array. The overhead
137 * is minimal in non-preemptive environments.
139 struct Lock FSYNC_handler_lock;
141 int FSYNC_clientInit(void)
143 struct sockaddr_in addr;
144 /* I can't believe the following is needed for localhost connections!! */
145 static backoff[] = {3,3,3,5,5,5,7,15,16,24,32,40,48,0};
146 int *timeout = &backoff[0];
149 FS_sd = getport(&addr);
150 if (connect(FS_sd, (struct sockaddr *) &addr, sizeof(addr)) >= 0)
152 #if defined(AFS_SGI_ENV)
153 /* down with worthless error messages! */
155 perror("FSYNC_clientInit failed (after many retries)");
162 perror("FSYNC_clientInit temporary failure (will retry)");
167 perror("FSYNC_clientInit failed (giving up!)");
171 void FSYNC_clientFinis(void)
179 Lock_Destroy(&FSYNC_handler_lock);
182 int FSYNC_askfs(VolumeId volume, char *partName, int com, int reason)
185 struct command command;
187 command.volume = volume;
188 command.command = com;
189 command.reason = reason;
191 strcpy(command.partName, partName);
193 command.partName[0] = 0;
196 if (send(FS_sd, (char*)&command, sizeof(command), 0) != sizeof(command)) {
197 printf("FSYNC_askfs: write to file server failed\n");
200 while ((n = recv(FS_sd, &response, 1, 0)) != 1) {
201 if (n == 0 || WSAEINTR != WSAGetLastError()) {
202 printf("FSYNC_askfs: No response from file server\n");
207 if (write(FS_sd, &command, sizeof(command)) != sizeof(command)) {
208 printf("FSYNC_askfs: write to file server failed\n");
211 while ((n = read(FS_sd, &response, 1)) != 1) {
212 if (n == 0 || errno != EINTR) {
213 printf("FSYNC_askfs: No response from file server\n");
219 printf("FSYNC_askfs: negative response from file server; volume %u, command %d\n", command.volume, command.command);
225 void FSYNC_fsInit(void)
227 #ifdef AFS_PTHREAD_ENV
229 pthread_attr_t tattr;
230 assert(pthread_attr_init(&tattr) == 0);
231 assert(pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED) == 0);
232 assert(pthread_create(&tid, &tattr, FSYNC_sync, NULL) == 0);
233 #else /* AFS_PTHREAD_ENV */
235 assert (LWP_CreateProcess(FSYNC_sync, USUAL_STACK_SIZE,
236 USUAL_PRIORITY, (void *) 0,
237 "FSYNC_sync", &pid) == LWP_SUCCESS);
238 #endif /* AFS_PTHREAD_ENV */
241 static int getport(struct sockaddr_in *addr)
245 memset(addr, 0, sizeof(*addr));
246 assert((sd = socket(AF_INET, SOCK_STREAM, 0)) >= 0);
247 #ifdef STRUCT_SOCKADDR_HAS_SA_LEN
248 addr->sin_len = sizeof(struct sockaddr_in);
250 addr->sin_addr.s_addr = htonl(0x7f000001);
251 addr->sin_family = AF_INET; /* was localhost->h_addrtype */
252 addr->sin_port = htons(2040); /* XXXX htons not _really_ neccessary */
257 static void FSYNC_sync() {
258 struct sockaddr_in addr;
263 #ifdef AFS_PTHREAD_ENV
268 signal(SIGPIPE, SIG_IGN);
271 #ifdef AFS_PTHREAD_ENV
272 /* set our 'thread-id' so that the host hold table works */
273 MUTEX_ENTER(&rx_stats_mutex); /* protects rxi_pthread_hinum */
274 tid=++rxi_pthread_hinum;
275 MUTEX_EXIT(&rx_stats_mutex);
276 pthread_setspecific(rx_thread_id_key, (void *)tid);
277 Log("Set thread id %d for FSYNC_sync\n", tid);
278 #endif /* AFS_PTHREAD_ENV */
281 /* Let somebody else run until level > 0. That doesn't mean that
282 * all volumes have been attached. */
283 #ifdef AFS_PTHREAD_ENV
285 #else /* AFS_PTHREAD_ENV */
286 LWP_DispatchProcess();
287 #endif /* AFS_PTHREAD_ENV */
289 AcceptSd = getport(&addr);
290 /* Reuseaddr needed because system inexplicably leaves crud lying around */
291 code = setsockopt(AcceptSd, SOL_SOCKET, SO_REUSEADDR, (char *) &on, sizeof(on));
292 if (code) Log("FSYNC_sync: setsockopt failed with (%d)\n",errno);
294 for (numTries=0; numTries < MAX_BIND_TRIES; numTries++) {
295 if ((code = bind(AcceptSd, (struct sockaddr *) &addr, sizeof(addr))) == 0) break;
296 Log("FSYNC_sync: bind failed with (%d), will sleep and retry\n",errno);
300 listen(AcceptSd,100);
306 GetHandler(&readfds, &maxfd);
307 /* Note: check for >= 1 below is essential since IOMGR_select
308 * doesn't have exactly same semantics as select.
310 #ifdef AFS_PTHREAD_ENV
311 if (select(maxfd+1, &readfds, NULL, NULL, NULL) >= 1)
312 #else /* AFS_PTHREAD_ENV */
313 if (IOMGR_Select(maxfd+1, &readfds, NULL, NULL, NULL) >= 1)
314 #endif /* AFS_PTHREAD_ENV */
315 CallHandler(&readfds);
319 static void FSYNC_newconnection(int afd)
321 struct sockaddr_in other;
323 junk = sizeof(other);
324 fd = accept(afd, (struct sockaddr *) &other, &junk);
326 Log("FSYNC_newconnection: accept failed, errno==%d\n", errno);
329 else if (!AddHandler(fd, FSYNC_com)) {
331 assert(AddHandler(fd, FSYNC_com));
339 afs_int32 FS_cnt = 0;
340 static void FSYNC_com(int fd)
345 struct command command;
347 register struct offlineInfo *volumes, *v;
349 char tvolName[VMAXPATHLEN];
353 n = recv(fd, &command, sizeof (command), 0);
355 n = read(fd, &command, sizeof (command));
361 if (n < sizeof(command)) {
362 Log("FSYNC_com: partial read (%d instead of %d); dropping connection (cnt=%d)\n", n, sizeof(command), FS_cnt);
368 volumes = OfflineVolumes[FindHandler(fd)];
369 for (v = 0, i = 0; i<MAXOFFLINEVOLUMES; i++) {
370 if (volumes[i].volumeID == command.volume
371 && strcmp(volumes[i].partName, command.partName)==0) {
376 switch (command.command) {
378 /* don't try to put online, this call is made only after deleting
379 a volume, in which case we want to remove the vol # from the
380 OfflineVolumes array only */
381 if (v) v->volumeID = 0;
386 This is where a detatched volume gets reattached. However in the
387 special case where the volume is merely busy, it is already
388 attatched and it is only necessary to clear the busy flag. See
389 defect #2080 for details.
392 /* is the volume already attatched? */
395 * XXX With the following enabled we had bizarre problems where the backup id would
396 * be reset to 0; that was due to the interaction between fileserver/volserver in that they
397 * both keep volumes in memory and the changes wouldn't be made to the fileserver. Some of
398 * the problems were due to refcnt changes as result of VGetVolume/VPutVolume which would call
399 * VOffline, etc. when we don't want to; someday the whole #2080 issue should be revisited to
402 vp=VGetVolume_r(&error,command.volume);
404 /* yep, is the BUSY flag set? */
405 if(vp->specialStatus==VBUSY) {
406 /* test harness for defect #2081 */
410 test #2081 by releasing TEST.2081,
411 so leave it alone here, zap it after
414 if(strcmp(vp->header->diskstuff.name,"TEST.2081")==0)
417 /* yep, clear BUSY flag */
420 /* make sure vol is online */
423 V_inUse(vp)=1; /* online */
432 /* so, we need to attach the volume */
437 sprintf(&tvolName[1], VFORMAT, command.volume);
439 vp = VAttachVolumeByName_r(&error, command.partName, tvolName, V_VOLUPD);
444 case FSYNC_NEEDVOLUME: {
446 /* not already offline, we need to find a slot for newly offline volume */
448 for (i = 0; i<MAXOFFLINEVOLUMES; i++) {
449 if (volumes[i].volumeID == 0) {
459 vp = VGetVolume_r(&error, command.volume);
461 if (command.partName[0] != 0
462 && strcmp(command.partName, vp->partition->name) != 0) {
463 /* volume on desired partition is not online, so we
464 * should treat this as an offline volume.
472 command.command==FSYNC_NEEDVOLUME
473 && (command.reason==V_READONLY
474 || (!VolumeWriteable(vp)
475 && (command.reason==V_CLONE || command.reason==V_DUMP))
479 if (command.command==FSYNC_NEEDVOLUME
480 && (command.reason==V_CLONE || command.reason==V_DUMP)) {
481 vp->specialStatus = VBUSY;
483 /* remember what volume we got, so we can keep track of how
484 * many volumes the volserver or whatever is using. Note that
485 * vp is valid since leaveonline is only set when vp is valid.
487 v->volumeID = command.volume;
488 strcpy(v->partName, vp->partition->name);
490 /* in this case, VOffline just returns sans decrementing
491 * ref count. We could try to fix it, but it has lots of
497 VOffline_r(vp, "A volume utility is running.");
502 VUpdateVolume_r(&error, vp); /* At least get volume stats right */
504 Log("FSYNC: Volume %u (%s) was left on line for an external %s request\n",
505 V_id(vp), V_name(vp),
506 command.reason == V_CLONE? "clone":
507 command.reason == V_READONLY? "readonly":
508 command.reason == V_DUMP? "dump" : "UNKNOWN");
517 case FSYNC_MOVEVOLUME:
518 /* Yuch: the "reason" for the move is the site it got moved to... */
519 /* still set specialStatus so we stop sending back VBUSY.
520 also should still break callbacks. Note that I don't know
521 how to tell if we should break all or not, so we just do it
522 since it doesn't matter much if we do an extra break
523 volume callbacks on a volume move within the same server */
524 vp = VGetVolume_r(&error, command.volume);
526 vp->specialStatus = VMOVED;
530 if (V_BreakVolumeCallbacks) {
531 Log("fssync: volume %u moved to %x; breaking all call backs\n",
532 command.volume, command.reason);
535 (*V_BreakVolumeCallbacks)(command.volume);
540 case FSYNC_RESTOREVOLUME:
541 /* if the volume is being restored, break all callbacks on it*/
542 if (V_BreakVolumeCallbacks) {
543 Log("fssync: volume %u restored; breaking all call backs\n",
547 (*V_BreakVolumeCallbacks)(command.volume);
565 static void FSYNC_Drop(int fd)
567 struct offlineInfo *p;
570 char tvolName[VMAXPATHLEN];
574 p = OfflineVolumes[FindHandler(fd)];
575 for (i = 0; i<MAXOFFLINEVOLUMES; i++) {
580 sprintf(&tvolName[1], VFORMAT, p[i].volumeID);
581 vp = VAttachVolumeByName_r(&error, p[i].partName, tvolName, V_VOLUPD);
598 static int AcceptHandler = -1; /* handler id for accept, if turned on */
600 static void AcceptOn() {
601 if (AcceptHandler == -1) {
602 assert(AddHandler(AcceptSd, FSYNC_newconnection));
603 AcceptHandler = FindHandler(AcceptSd);
607 static void AcceptOff() {
608 if (AcceptHandler != -1) {
609 assert(RemoveHandler(AcceptSd));
614 /* The multiple FD handling code. */
616 static int HandlerFD[MAXHANDLERS];
617 static int (*HandlerProc[MAXHANDLERS])();
619 static void InitHandler ()
622 ObtainWriteLock(&FSYNC_handler_lock);
623 for(i=0;i<MAXHANDLERS;i++)
627 ReleaseWriteLock(&FSYNC_handler_lock);
630 static void CallHandler(fd_set *fdsetp)
633 ObtainReadLock(&FSYNC_handler_lock);
634 for(i=0;i<MAXHANDLERS;i++) {
635 if (HandlerFD[i] >= 0 && FD_ISSET(HandlerFD[i], fdsetp)) {
636 ReleaseReadLock(&FSYNC_handler_lock);
637 (*HandlerProc[i])(HandlerFD[i]);
638 ObtainReadLock(&FSYNC_handler_lock);
641 ReleaseReadLock(&FSYNC_handler_lock);
644 static int AddHandler (int afd, int (*aproc)())
647 ObtainWriteLock(&FSYNC_handler_lock);
648 for(i=0;i<MAXHANDLERS;i++)
649 if (HandlerFD[i] == -1) break;
650 if (i>=MAXHANDLERS) {
651 ReleaseWriteLock(&FSYNC_handler_lock);
655 HandlerProc[i] = aproc;
656 ReleaseWriteLock(&FSYNC_handler_lock);
660 static int FindHandler(register int afd)
663 ObtainReadLock(&FSYNC_handler_lock);
664 for(i=0;i<MAXHANDLERS;i++)
665 if (HandlerFD[i] == afd) {
666 ReleaseReadLock(&FSYNC_handler_lock);
669 ReleaseReadLock(&FSYNC_handler_lock); /* just in case */
671 return -1; /* satisfy compiler */
674 static int FindHandler_r(register int afd)
677 for(i=0;i<MAXHANDLERS;i++)
678 if (HandlerFD[i] == afd) {
682 return -1; /* satisfy compiler */
685 static int RemoveHandler(register int afd)
687 ObtainWriteLock(&FSYNC_handler_lock);
688 HandlerFD[FindHandler_r(afd)] = -1;
689 ReleaseWriteLock(&FSYNC_handler_lock);
693 static void GetHandler(fd_set *fdsetp, int *maxfdp)
696 register int maxfd = -1;
698 ObtainReadLock(&FSYNC_handler_lock); /* just in case */
699 for(i=0;i<MAXHANDLERS;i++)
700 if (HandlerFD[i] != -1) {
701 FD_SET(HandlerFD[i], fdsetp);
702 if (maxfd < HandlerFD[i])
703 maxfd = HandlerFD[i];
706 ReleaseReadLock(&FSYNC_handler_lock); /* just in case */