2 * Copyright 2000, International Business Machines Corporation and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
10 #include <afs/param.h>
24 osi_rwlock_t cm_serverLock;
26 cm_server_t *cm_allServersp;
29 cm_ForceNewConnectionsAllServers(void)
33 lock_ObtainRead(&cm_serverLock);
34 for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
35 cm_GetServerNoLock(tsp);
36 cm_ForceNewConnections(tsp);
37 cm_PutServerNoLock(tsp);
39 lock_ReleaseRead(&cm_serverLock);
43 cm_PingServer(cm_server_t *tsp)
48 struct rx_connection * rxconnp;
51 Capabilities caps = {0, 0};
55 lock_ObtainMutex(&tsp->mx);
56 if (tsp->flags & CM_SERVERFLAG_PINGING) {
58 osi_SleepM((LONG_PTR)tsp, &tsp->mx);
59 lock_ObtainMutex(&tsp->mx);
61 if (tsp->waitCount == 0)
62 tsp->flags &= ~CM_SERVERFLAG_PINGING;
64 osi_Wakeup((LONG_PTR)tsp);
65 lock_ReleaseMutex(&tsp->mx);
68 tsp->flags |= CM_SERVERFLAG_PINGING;
69 wasDown = tsp->flags & CM_SERVERFLAG_DOWN;
70 afs_inet_ntoa_r(tsp->addr.sin_addr.S_un.S_addr, hoststr);
71 lock_ReleaseMutex(&tsp->mx);
73 code = cm_ConnByServer(tsp, cm_rootUserp, &connp);
75 /* now call the appropriate ping call. Drop the timeout if
76 * the server is known to be down, so that we don't waste a
77 * lot of time retiming out down servers.
80 osi_Log4(afsd_logp, "cm_PingServer server %s (%s) was %s with caps 0x%x",
81 osi_LogSaveString(afsd_logp, hoststr),
82 tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
83 wasDown ? "down" : "up",
87 rx_SetConnDeadTime(connp->callp, 10);
88 if (tsp->type == CM_SERVER_VLDB) {
89 code = VL_ProbeServer(connp->callp);
93 rxconnp = cm_GetRxConn(connp);
94 code = RXAFS_GetCapabilities(rxconnp, &caps);
95 if (code == RXGEN_OPCODE)
96 code = RXAFS_GetTime(rxconnp, &secs, &usecs);
97 rx_PutConnection(rxconnp);
100 rx_SetConnDeadTime(connp->callp, ConnDeadtimeout);
102 } /* got an unauthenticated connection to this server */
104 lock_ObtainMutex(&tsp->mx);
106 /* mark server as up */
107 tsp->flags &= ~CM_SERVERFLAG_DOWN;
109 /* we currently handle 32-bits of capabilities */
110 if (caps.Capabilities_len > 0) {
111 tsp->capabilities = caps.Capabilities_val[0];
112 free(caps.Capabilities_val);
113 caps.Capabilities_len = 0;
114 caps.Capabilities_val = 0;
116 tsp->capabilities = 0;
119 osi_Log3(afsd_logp, "cm_PingServer server %s (%s) is up with caps 0x%x",
120 osi_LogSaveString(afsd_logp, hoststr),
121 tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
124 /* Now update the volume status if necessary */
126 cm_server_vols_t * tsrvp;
130 lock_ReleaseMutex(&tsp->mx);
131 for (tsrvp = tsp->vols; tsrvp; tsrvp = tsrvp->nextp) {
132 for (i=0; i<NUM_SERVER_VOLS; i++) {
133 if (tsrvp->ids[i] != 0) {
136 code = cm_GetVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
137 &req, CM_GETVOL_FLAG_NO_LRU_UPDATE, &volp);
139 cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
145 lock_ObtainMutex(&tsp->mx);
148 /* mark server as down */
149 tsp->flags |= CM_SERVERFLAG_DOWN;
150 if (code != VRESTARTING)
151 cm_ForceNewConnections(tsp);
153 osi_Log3(afsd_logp, "cm_PingServer server %s (%s) is down with caps 0x%x",
154 osi_LogSaveString(afsd_logp, hoststr),
155 tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
158 /* Now update the volume status if necessary */
160 cm_server_vols_t * tsrvp;
164 lock_ReleaseMutex(&tsp->mx);
165 for (tsrvp = tsp->vols; tsrvp; tsrvp = tsrvp->nextp) {
166 for (i=0; i<NUM_SERVER_VOLS; i++) {
167 if (tsrvp->ids[i] != 0) {
170 code = cm_GetVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
171 &req, CM_GETVOL_FLAG_NO_LRU_UPDATE, &volp);
173 cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
179 lock_ObtainMutex(&tsp->mx);
183 if (tsp->waitCount == 0)
184 tsp->flags &= ~CM_SERVERFLAG_PINGING;
186 osi_Wakeup((LONG_PTR)tsp);
187 lock_ReleaseMutex(&tsp->mx);
191 void cm_CheckServers(long flags, cm_cell_t *cellp)
193 /* ping all file servers, up or down, with unauthenticated connection,
194 * to find out whether we have all our callbacks from the server still.
195 * Also, ping down VLDBs.
202 lock_ObtainWrite(&cm_serverLock);
203 for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
204 cm_GetServerNoLock(tsp);
205 lock_ReleaseWrite(&cm_serverLock);
207 /* now process the server */
208 lock_ObtainMutex(&tsp->mx);
211 isDown = tsp->flags & CM_SERVERFLAG_DOWN;
212 isFS = tsp->type == CM_SERVER_FILE;
214 /* only do the ping if the cell matches the requested cell, or we're
215 * matching all cells (cellp == NULL), and if we've requested to ping
216 * this type of {up, down} servers.
218 if ((cellp == NULL || cellp == tsp->cellp) &&
219 ((isDown && (flags & CM_FLAG_CHECKDOWNSERVERS)) ||
220 (!isDown && (flags & CM_FLAG_CHECKUPSERVERS))) &&
221 ((!(flags & CM_FLAG_CHECKVLDBSERVERS) ||
222 !isFS && (flags & CM_FLAG_CHECKVLDBSERVERS)) &&
223 (!(flags & CM_FLAG_CHECKFILESERVERS) ||
224 isFS && (flags & CM_FLAG_CHECKFILESERVERS)))) {
226 } /* we're supposed to check this up/down server */
227 lock_ReleaseMutex(&tsp->mx);
229 /* at this point, we've adjusted the server state, so do the ping and
235 #ifdef GIVE_UP_CALLBACKS
236 /* if this is a file server and it is not currently down
237 * give up any callbacks we have queued
239 if (isFS && !(tsp->flags & CM_SERVERFLAG_DOWN)) {
240 lock_ObtainMutex(&tsp->mx);
241 cm_GiveUpCallBacksToServer(tsp);
242 lock_ReleaseMutex(&tsp->mx);
244 #endif /* GIVE_UP_CALLBACKS */
246 /* also, run the GC function for connections on all of the
247 * server's connections.
249 cm_GCConnections(tsp);
251 lock_ObtainWrite(&cm_serverLock);
252 cm_PutServerNoLock(tsp);
254 lock_ReleaseWrite(&cm_serverLock);
257 void cm_InitServer(void)
259 static osi_once_t once;
261 if (osi_Once(&once)) {
262 lock_InitializeRWLock(&cm_serverLock, "cm_serverLock");
267 void cm_GetServer(cm_server_t *serverp)
269 lock_ObtainWrite(&cm_serverLock);
271 lock_ReleaseWrite(&cm_serverLock);
274 void cm_GetServerNoLock(cm_server_t *serverp)
279 void cm_PutServer(cm_server_t *serverp)
281 lock_ObtainWrite(&cm_serverLock);
282 osi_assert(serverp->refCount-- > 0);
283 lock_ReleaseWrite(&cm_serverLock);
286 void cm_PutServerNoLock(cm_server_t *serverp)
288 osi_assert(serverp->refCount-- > 0);
291 void cm_SetServerNo64Bit(cm_server_t * serverp, int no64bit)
293 lock_ObtainMutex(&serverp->mx);
295 serverp->flags |= CM_SERVERFLAG_NO64BIT;
297 serverp->flags &= ~CM_SERVERFLAG_NO64BIT;
298 lock_ReleaseMutex(&serverp->mx);
301 void cm_SetServerNoInlineBulk(cm_server_t * serverp, int no)
303 lock_ObtainMutex(&serverp->mx);
305 serverp->flags |= CM_SERVERFLAG_NOINLINEBULK;
307 serverp->flags &= ~CM_SERVERFLAG_NOINLINEBULK;
308 lock_ReleaseMutex(&serverp->mx);
311 void cm_SetServerPrefs(cm_server_t * serverp)
313 unsigned long serverAddr; /* in host byte order */
314 unsigned long myAddr, myNet, mySubnet;/* in host byte order */
315 unsigned long netMask;
318 int cm_noIPAddr; /* number of client network interfaces */
319 int cm_IPAddr[CM_MAXINTERFACE_ADDR]; /* client's IP address in host order */
320 int cm_SubnetMask[CM_MAXINTERFACE_ADDR];/* client's subnet mask in host order*/
321 int cm_NetMtu[CM_MAXINTERFACE_ADDR]; /* client's MTU sizes */
322 int cm_NetFlags[CM_MAXINTERFACE_ADDR]; /* network flags */
325 /* get network related info */
326 cm_noIPAddr = CM_MAXINTERFACE_ADDR;
327 code = syscfg_GetIFInfo(&cm_noIPAddr,
328 cm_IPAddr, cm_SubnetMask,
329 cm_NetMtu, cm_NetFlags);
331 serverAddr = ntohl(serverp->addr.sin_addr.s_addr);
332 serverp->ipRank = CM_IPRANK_LOW; /* default setings */
334 for ( i=0; i < cm_noIPAddr; i++)
336 /* loop through all the client's IP address and compare
337 ** each of them against the server's IP address */
339 myAddr = cm_IPAddr[i];
340 if ( IN_CLASSA(myAddr) )
341 netMask = IN_CLASSA_NET;
342 else if ( IN_CLASSB(myAddr) )
343 netMask = IN_CLASSB_NET;
344 else if ( IN_CLASSC(myAddr) )
345 netMask = IN_CLASSC_NET;
349 myNet = myAddr & netMask;
350 mySubnet = myAddr & cm_SubnetMask[i];
352 if ( (serverAddr & netMask) == myNet )
354 if ( (serverAddr & cm_SubnetMask[i]) == mySubnet)
356 if ( serverAddr == myAddr )
357 serverp->ipRank = min(serverp->ipRank,
358 CM_IPRANK_TOP);/* same machine */
359 else serverp->ipRank = min(serverp->ipRank,
360 CM_IPRANK_HI); /* same subnet */
362 else serverp->ipRank = min(serverp->ipRank,CM_IPRANK_MED);
365 /* random between 0..15*/
366 serverp->ipRank += min(serverp->ipRank, rand() % 0x000f);
367 } /* and of for loop */
370 cm_server_t *cm_NewServer(struct sockaddr_in *socketp, int type, cm_cell_t *cellp) {
373 osi_assert(socketp->sin_family == AF_INET);
375 tsp = malloc(sizeof(*tsp));
377 memset(tsp, 0, sizeof(*tsp));
381 lock_InitializeMutex(&tsp->mx, "cm_server_t mutex");
382 tsp->addr = *socketp;
383 tsp->flags = CM_SERVERFLAG_DOWN; /* assume down; ping will mark up if available */
385 cm_SetServerPrefs(tsp);
387 lock_ObtainWrite(&cm_serverLock); /* get server lock */
388 tsp->allNextp = cm_allServersp;
389 cm_allServersp = tsp;
390 lock_ReleaseWrite(&cm_serverLock); /* release server lock */
392 cm_PingServer(tsp); /* Obtain Capabilities and check up/down state */
398 cm_FindServerByIP(afs_uint32 ipaddr, int type)
402 lock_ObtainRead(&cm_serverLock);
403 for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
404 if (tsp->type == type &&
405 tsp->addr.sin_addr.S_un.S_addr == ipaddr)
408 lock_ReleaseRead(&cm_serverLock);
413 /* find a server based on its properties */
414 cm_server_t *cm_FindServer(struct sockaddr_in *addrp, int type)
418 osi_assert(addrp->sin_family == AF_INET);
420 lock_ObtainWrite(&cm_serverLock);
421 for (tsp = cm_allServersp; tsp; tsp=tsp->allNextp) {
422 if (tsp->type == type &&
423 tsp->addr.sin_addr.s_addr == addrp->sin_addr.s_addr)
427 /* bump ref count if we found the server */
429 cm_GetServerNoLock(tsp);
431 /* drop big table lock */
432 lock_ReleaseWrite(&cm_serverLock);
434 /* return what we found */
438 cm_server_vols_t *cm_NewServerVols(void) {
439 cm_server_vols_t *tsvp;
441 tsvp = malloc(sizeof(*tsvp));
443 memset(tsvp, 0, sizeof(*tsvp));
448 cm_serverRef_t *cm_NewServerRef(cm_server_t *serverp, afs_uint32 volID)
450 cm_serverRef_t *tsrp;
451 cm_server_vols_t **tsrvpp = NULL;
452 afs_uint32 *slotp = NULL;
455 cm_GetServer(serverp);
456 tsrp = malloc(sizeof(*tsrp));
457 tsrp->server = serverp;
458 tsrp->status = srv_not_busy;
463 /* if we have a non-zero volID, we need to add it to the list
464 * of volumes maintained by the server. There are two phases:
465 * (1) see if the volID is already in the list and (2) insert
466 * it into the first empty slot if it is not.
469 lock_ObtainMutex(&serverp->mx);
471 tsrvpp = &serverp->vols;
475 for (i=0; i<NUM_SERVER_VOLS; i++) {
476 if ((*tsrvpp)->ids[i] == volID) {
479 } else if (!slotp && (*tsrvpp)->ids[i] == 0) {
480 slotp = &(*tsrvpp)->ids[i];
487 tsrvpp = &(*tsrvpp)->nextp;
494 /* if we didn't find an empty slot in a current
495 * page we must need a new page */
496 *tsrvpp = cm_NewServerVols();
498 (*tsrvpp)->ids[0] = volID;
502 lock_ReleaseMutex(&serverp->mx);
508 LONG_PTR cm_ChecksumServerList(cm_serverRef_t *serversp)
512 cm_serverRef_t *tsrp;
514 lock_ObtainWrite(&cm_serverLock);
515 for (tsrp = serversp; tsrp; tsrp=tsrp->next) {
520 sum ^= (LONG_PTR) tsrp->server;
523 lock_ReleaseWrite(&cm_serverLock);
528 ** Insert a server into the server list keeping the list sorted in
529 ** asending order of ipRank.
531 ** The refCount of the cm_serverRef_t is increased
533 void cm_InsertServerList(cm_serverRef_t** list, cm_serverRef_t* element)
535 cm_serverRef_t *current=*list;
536 unsigned short ipRank = element->server->ipRank;
538 lock_ObtainWrite(&cm_serverLock);
539 element->refCount++; /* increase refCount */
541 /* insertion into empty list or at the beginning of the list */
542 if ( !current || (current->server->ipRank > ipRank) )
544 element->next = *list;
546 lock_ReleaseWrite(&cm_serverLock);
550 while ( current->next ) /* find appropriate place to insert */
552 if ( current->next->server->ipRank > ipRank )
554 else current = current->next;
556 element->next = current->next;
557 current->next = element;
558 lock_ReleaseWrite(&cm_serverLock);
561 ** Re-sort the server list with the modified rank
562 ** returns 0 if element was changed successfully.
563 ** returns 1 if list remained unchanged.
565 long cm_ChangeRankServer(cm_serverRef_t** list, cm_server_t* server)
567 cm_serverRef_t **current=list;
568 cm_serverRef_t *element=0;
570 /* if there is max of one element in the list, nothing to sort */
571 if ( (!*current) || !((*current)->next) )
572 return 1; /* list unchanged: return success */
574 lock_ObtainWrite(&cm_serverLock);
575 /* if the server is on the list, delete it from list */
578 if ( (*current)->server == server)
580 element = (*current);
581 *current = (*current)->next; /* delete it */
584 current = & ( (*current)->next);
586 lock_ReleaseWrite(&cm_serverLock);
588 /* if this volume is not replicated on this server */
590 return 1; /* server is not on list */
592 /* re-insert deleted element into the list with modified rank*/
593 cm_InsertServerList(list, element);
595 /* reduce refCount which was increased by cm_InsertServerList */
596 lock_ObtainWrite(&cm_serverLock);
598 lock_ReleaseWrite(&cm_serverLock);
602 ** If there are more than one server on the list and the first n servers on
603 ** the list have the same rank( n>1), then randomise among the first n servers.
605 void cm_RandomizeServer(cm_serverRef_t** list)
608 cm_serverRef_t* tsrp = *list, *lastTsrp;
609 unsigned short lowestRank;
611 /* an empty list or a list with only one element */
612 if ( !tsrp || ! tsrp->next )
615 lock_ObtainWrite(&cm_serverLock);
617 /* count the number of servers with the lowest rank */
618 lowestRank = tsrp->server->ipRank;
619 for ( count=1, tsrp=tsrp->next; tsrp; tsrp=tsrp->next)
621 if ( tsrp->server->ipRank != lowestRank)
627 /* if there is only one server with the lowest rank, we are done */
629 lock_ReleaseWrite(&cm_serverLock);
633 picked = rand() % count;
635 lock_ReleaseWrite(&cm_serverLock);
640 while (--picked >= 0)
645 lastTsrp->next = tsrp->next; /* delete random element from list*/
646 tsrp->next = *list; /* insert element at the beginning of list */
648 lock_ReleaseWrite(&cm_serverLock);
651 /* call cm_FreeServer while holding a write lock on cm_serverLock */
652 void cm_FreeServer(cm_server_t* serverp)
654 cm_server_vols_t * tsrvp, *nextp;
656 cm_PutServerNoLock(serverp);
657 if (serverp->refCount == 0)
659 /* we need to check to ensure that all of the connections
660 * for this server have a 0 refCount; otherwise, they will
661 * not be garbage collected
663 cm_GCConnections(serverp); /* connsp */
665 if (!(serverp->flags & CM_SERVERFLAG_PREF_SET)) {
666 lock_FinalizeMutex(&serverp->mx);
667 if ( cm_allServersp == serverp )
668 cm_allServersp = serverp->allNextp;
672 for(tsp = cm_allServersp; tsp->allNextp; tsp=tsp->allNextp) {
673 if ( tsp->allNextp == serverp ) {
674 tsp->allNextp = serverp->allNextp;
680 /* free the volid list */
681 for ( tsrvp = serverp->vols; tsrvp; tsrvp = nextp) {
682 nextp = tsrvp->nextp;
691 void cm_RemoveVolumeFromServer(cm_server_t * serverp, afs_uint32 volID)
693 cm_server_vols_t * tsrvp;
699 for (tsrvp = serverp->vols; tsrvp; tsrvp = tsrvp->nextp) {
700 for (i=0; i<NUM_SERVER_VOLS; i++) {
701 if (tsrvp->ids[i] == volID) {
709 void cm_FreeServerList(cm_serverRef_t** list, afs_uint32 flags)
711 cm_serverRef_t **current = list;
712 cm_serverRef_t **nextp = 0;
713 cm_serverRef_t * next = 0;
715 lock_ObtainWrite(&cm_serverLock);
719 nextp = &(*current)->next;
720 if (--((*current)->refCount) == 0) {
723 if ((*current)->volID)
724 cm_RemoveVolumeFromServer((*current)->server, (*current)->volID);
725 cm_FreeServer((*current)->server);
729 if (flags & CM_FREESERVERLIST_DELETE) {
730 (*current)->status = srv_deleted;
731 if ((*current)->volID)
732 cm_RemoveVolumeFromServer((*current)->server, (*current)->volID);
738 lock_ReleaseWrite(&cm_serverLock);
741 #ifdef GIVE_UP_CALLBACKS
742 cm_server_gucb_t *cm_NewServerGUCBs(void) {
743 cm_server_gucb_t *gucbp;
745 gucbp = malloc(sizeof(*gucbp));
747 memset(gucbp, 0, sizeof(*gucbp));
753 /* server mutex must be held */
754 void cm_AddFidToGiveUpCallBackList(cm_server_t * serverp, cm_fid_t *fidp) {
755 cm_server_gucb_t ** gucbpp;
757 for ( gucbpp = &serverp->gucbs; *gucbpp; gucbpp = &(*gucbpp)->nextp ) {
758 if ((*gucbpp)->count < AFS_MAXCBRSCALL) {
759 (*gucbpp)->fids[(*gucbpp)->count] = *fidp;
765 /* if we get here all of the allocated pages are full */
766 (*gucbpp) = cm_NewServerGUCBs();
768 (*gucbpp)->fids[0] = *fidp;
769 (*gucbpp)->count = 1;
773 /* server mutex must be held */
774 void cm_RemoveFidFromGiveUpCallBackList(cm_server_t *serverp, cm_fid_t *fidp) {
775 cm_server_gucb_t *gucbp;
778 for ( gucbp = serverp->gucbs; gucbp; gucbp = gucbp->nextp ) {
779 for ( i=0; i < gucbp->count; i++ ) {
780 if (cm_FidCmp(&gucbp->fids[i], fidp) == 0) {
781 /* invalidate this entry. we will skip over it later */
782 gucbp->fids[i].cell = 0;
789 /* server mutex must be held */
790 void cm_FreeGiveUpCallBackList(cm_server_t * serverp)
792 cm_server_gucb_t *gucbp, *nextp;
794 for ( gucbp = serverp->gucbs, serverp->gucbs = NULL; gucbp; gucbp = nextp ) {
795 nextp = gucbp->nextp;
800 void cm_FreeAllGiveUpCallBackLists(void)
804 lock_ObtainRead(&cm_serverLock);
805 for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
806 if (tsp->type == CM_SERVER_FILE && tsp->gucbs != NULL) {
807 lock_ObtainMutex(&tsp->mx);
808 cm_FreeGiveUpCallBackList(tsp);
809 lock_ReleaseMutex(&tsp->mx);
812 lock_ReleaseRead(&cm_serverLock);
814 #endif /* GIVE_UP_CALLBACKS */