2 * Copyright 2000, International Business Machines Corporation and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
10 #include <afs/param.h>
18 #include <sys/socket.h>
25 #include <WINNT\syscfg.h>
29 osi_rwlock_t cm_serverLock;
31 cm_server_t *cm_allServersp;
34 cm_ForceNewConnectionsAllServers(void)
38 lock_ObtainWrite(&cm_serverLock);
39 for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
40 cm_GetServerNoLock(tsp);
41 cm_ForceNewConnections(tsp);
42 cm_PutServerNoLock(tsp);
44 lock_ReleaseWrite(&cm_serverLock);
48 cm_PingServer(cm_server_t *tsp)
53 struct rx_connection * rxconnp;
56 Capabilities caps = {0, 0};
60 lock_ObtainMutex(&tsp->mx);
61 if (tsp->flags & CM_SERVERFLAG_PINGING) {
63 osi_SleepM((LONG_PTR)tsp, &tsp->mx);
64 lock_ObtainMutex(&tsp->mx);
66 if (tsp->waitCount == 0)
67 tsp->flags &= ~CM_SERVERFLAG_PINGING;
69 osi_Wakeup((LONG_PTR)tsp);
70 lock_ReleaseMutex(&tsp->mx);
73 tsp->flags |= CM_SERVERFLAG_PINGING;
74 wasDown = tsp->flags & CM_SERVERFLAG_DOWN;
75 afs_inet_ntoa_r(tsp->addr.sin_addr.S_un.S_addr, hoststr);
76 lock_ReleaseMutex(&tsp->mx);
78 code = cm_ConnByServer(tsp, cm_rootUserp, &connp);
80 /* now call the appropriate ping call. Drop the timeout if
81 * the server is known to be down, so that we don't waste a
82 * lot of time retiming out down servers.
85 osi_Log4(afsd_logp, "cm_PingServer server %s (%s) was %s with caps 0x%x",
86 osi_LogSaveString(afsd_logp, hoststr),
87 tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
88 wasDown ? "down" : "up",
91 rxconnp = cm_GetRxConn(connp);
93 rx_SetConnDeadTime(rxconnp, 10);
94 if (tsp->type == CM_SERVER_VLDB) {
95 code = VL_ProbeServer(rxconnp);
99 code = RXAFS_GetCapabilities(rxconnp, &caps);
100 if (code == RXGEN_OPCODE)
101 code = RXAFS_GetTime(rxconnp, &secs, &usecs);
104 rx_SetConnDeadTime(rxconnp, ConnDeadtimeout);
105 rx_PutConnection(rxconnp);
107 } /* got an unauthenticated connection to this server */
109 lock_ObtainMutex(&tsp->mx);
111 /* mark server as up */
112 tsp->flags &= ~CM_SERVERFLAG_DOWN;
115 /* we currently handle 32-bits of capabilities */
116 if (caps.Capabilities_len > 0) {
117 tsp->capabilities = caps.Capabilities_val[0];
118 free(caps.Capabilities_val);
119 caps.Capabilities_len = 0;
120 caps.Capabilities_val = 0;
122 tsp->capabilities = 0;
125 osi_Log3(afsd_logp, "cm_PingServer server %s (%s) is up with caps 0x%x",
126 osi_LogSaveString(afsd_logp, hoststr),
127 tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
130 /* Now update the volume status if necessary */
132 cm_server_vols_t * tsrvp;
136 for (tsrvp = tsp->vols; tsrvp; tsrvp = tsrvp->nextp) {
137 for (i=0; i<NUM_SERVER_VOLS; i++) {
138 if (tsrvp->ids[i] != 0) {
141 lock_ReleaseMutex(&tsp->mx);
142 code = cm_GetVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
143 &req, CM_GETVOL_FLAG_NO_LRU_UPDATE, &volp);
144 lock_ObtainMutex(&tsp->mx);
146 cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
154 /* mark server as down */
155 if (!(tsp->flags & CM_SERVERFLAG_DOWN)) {
156 tsp->flags |= CM_SERVERFLAG_DOWN;
157 tsp->downTime = osi_Time();
159 if (code != VRESTARTING)
160 cm_ForceNewConnections(tsp);
162 osi_Log3(afsd_logp, "cm_PingServer server %s (%s) is down with caps 0x%x",
163 osi_LogSaveString(afsd_logp, hoststr),
164 tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
167 /* Now update the volume status if necessary */
169 cm_server_vols_t * tsrvp;
173 for (tsrvp = tsp->vols; tsrvp; tsrvp = tsrvp->nextp) {
174 for (i=0; i<NUM_SERVER_VOLS; i++) {
175 if (tsrvp->ids[i] != 0) {
178 lock_ReleaseMutex(&tsp->mx);
179 code = cm_GetVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
180 &req, CM_GETVOL_FLAG_NO_LRU_UPDATE, &volp);
181 lock_ObtainMutex(&tsp->mx);
183 cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
192 if (tsp->waitCount == 0)
193 tsp->flags &= ~CM_SERVERFLAG_PINGING;
195 osi_Wakeup((LONG_PTR)tsp);
196 lock_ReleaseMutex(&tsp->mx);
200 void cm_CheckServers(long flags, cm_cell_t *cellp)
202 /* ping all file servers, up or down, with unauthenticated connection,
203 * to find out whether we have all our callbacks from the server still.
204 * Also, ping down VLDBs.
211 lock_ObtainWrite(&cm_serverLock);
212 for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
213 cm_GetServerNoLock(tsp);
214 lock_ReleaseWrite(&cm_serverLock);
216 /* now process the server */
217 lock_ObtainMutex(&tsp->mx);
220 isDown = tsp->flags & CM_SERVERFLAG_DOWN;
221 isFS = tsp->type == CM_SERVER_FILE;
223 /* only do the ping if the cell matches the requested cell, or we're
224 * matching all cells (cellp == NULL), and if we've requested to ping
225 * this type of {up, down} servers.
227 if ((cellp == NULL || cellp == tsp->cellp) &&
228 ((isDown && (flags & CM_FLAG_CHECKDOWNSERVERS)) ||
229 (!isDown && (flags & CM_FLAG_CHECKUPSERVERS))) &&
230 ((!(flags & CM_FLAG_CHECKVLDBSERVERS) ||
231 !isFS && (flags & CM_FLAG_CHECKVLDBSERVERS)) &&
232 (!(flags & CM_FLAG_CHECKFILESERVERS) ||
233 isFS && (flags & CM_FLAG_CHECKFILESERVERS)))) {
235 } /* we're supposed to check this up/down server */
236 lock_ReleaseMutex(&tsp->mx);
238 /* at this point, we've adjusted the server state, so do the ping and
244 /* also, run the GC function for connections on all of the
245 * server's connections.
247 cm_GCConnections(tsp);
249 lock_ObtainWrite(&cm_serverLock);
250 cm_PutServerNoLock(tsp);
252 lock_ReleaseWrite(&cm_serverLock);
255 void cm_InitServer(void)
257 static osi_once_t once;
259 if (osi_Once(&once)) {
260 lock_InitializeRWLock(&cm_serverLock, "cm_serverLock");
265 void cm_GetServer(cm_server_t *serverp)
267 lock_ObtainWrite(&cm_serverLock);
269 lock_ReleaseWrite(&cm_serverLock);
272 void cm_GetServerNoLock(cm_server_t *serverp)
277 void cm_PutServer(cm_server_t *serverp)
279 lock_ObtainWrite(&cm_serverLock);
280 osi_assertx(serverp->refCount-- > 0, "cm_server_t refCount 0");
281 lock_ReleaseWrite(&cm_serverLock);
284 void cm_PutServerNoLock(cm_server_t *serverp)
286 osi_assertx(serverp->refCount-- > 0, "cm_server_t refCount 0");
289 void cm_SetServerNo64Bit(cm_server_t * serverp, int no64bit)
291 lock_ObtainMutex(&serverp->mx);
293 serverp->flags |= CM_SERVERFLAG_NO64BIT;
295 serverp->flags &= ~CM_SERVERFLAG_NO64BIT;
296 lock_ReleaseMutex(&serverp->mx);
299 void cm_SetServerNoInlineBulk(cm_server_t * serverp, int no)
301 lock_ObtainMutex(&serverp->mx);
303 serverp->flags |= CM_SERVERFLAG_NOINLINEBULK;
305 serverp->flags &= ~CM_SERVERFLAG_NOINLINEBULK;
306 lock_ReleaseMutex(&serverp->mx);
309 void cm_SetServerPrefs(cm_server_t * serverp)
311 unsigned long serverAddr; /* in host byte order */
312 unsigned long myAddr, myNet, mySubnet;/* in host byte order */
313 unsigned long netMask;
316 int cm_noIPAddr; /* number of client network interfaces */
317 int cm_IPAddr[CM_MAXINTERFACE_ADDR]; /* client's IP address in host order */
318 int cm_SubnetMask[CM_MAXINTERFACE_ADDR];/* client's subnet mask in host order*/
319 int cm_NetMtu[CM_MAXINTERFACE_ADDR]; /* client's MTU sizes */
320 int cm_NetFlags[CM_MAXINTERFACE_ADDR]; /* network flags */
323 /* get network related info */
324 cm_noIPAddr = CM_MAXINTERFACE_ADDR;
325 code = syscfg_GetIFInfo(&cm_noIPAddr,
326 cm_IPAddr, cm_SubnetMask,
327 cm_NetMtu, cm_NetFlags);
329 serverAddr = ntohl(serverp->addr.sin_addr.s_addr);
330 serverp->ipRank = CM_IPRANK_LOW; /* default setings */
332 for ( i=0; i < cm_noIPAddr; i++)
334 /* loop through all the client's IP address and compare
335 ** each of them against the server's IP address */
337 myAddr = cm_IPAddr[i];
338 if ( IN_CLASSA(myAddr) )
339 netMask = IN_CLASSA_NET;
340 else if ( IN_CLASSB(myAddr) )
341 netMask = IN_CLASSB_NET;
342 else if ( IN_CLASSC(myAddr) )
343 netMask = IN_CLASSC_NET;
347 myNet = myAddr & netMask;
348 mySubnet = myAddr & cm_SubnetMask[i];
350 if ( (serverAddr & netMask) == myNet )
352 if ( (serverAddr & cm_SubnetMask[i]) == mySubnet)
354 if ( serverAddr == myAddr )
355 serverp->ipRank = min(serverp->ipRank,
356 CM_IPRANK_TOP);/* same machine */
357 else serverp->ipRank = min(serverp->ipRank,
358 CM_IPRANK_HI); /* same subnet */
360 else serverp->ipRank = min(serverp->ipRank,CM_IPRANK_MED);
363 /* random between 0..15*/
364 serverp->ipRank += min(serverp->ipRank, rand() % 0x000f);
365 } /* and of for loop */
368 cm_server_t *cm_NewServer(struct sockaddr_in *socketp, int type, cm_cell_t *cellp, afs_uint32 flags) {
371 osi_assertx(socketp->sin_family == AF_INET, "unexpected socket family");
373 tsp = malloc(sizeof(*tsp));
375 memset(tsp, 0, sizeof(*tsp));
379 lock_InitializeMutex(&tsp->mx, "cm_server_t mutex");
380 tsp->addr = *socketp;
381 tsp->flags = CM_SERVERFLAG_DOWN; /* assume down; ping will mark up if available */
383 cm_SetServerPrefs(tsp);
385 lock_ObtainWrite(&cm_serverLock); /* get server lock */
386 tsp->allNextp = cm_allServersp;
387 cm_allServersp = tsp;
388 lock_ReleaseWrite(&cm_serverLock); /* release server lock */
390 if ( !(flags & CM_FLAG_NOPROBE) )
391 cm_PingServer(tsp); /* Obtain Capabilities and check up/down state */
397 cm_FindServerByIP(afs_uint32 ipaddr, int type)
401 lock_ObtainRead(&cm_serverLock);
402 for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
403 if (tsp->type == type &&
404 tsp->addr.sin_addr.S_un.S_addr == ipaddr)
407 lock_ReleaseRead(&cm_serverLock);
412 /* find a server based on its properties */
413 cm_server_t *cm_FindServer(struct sockaddr_in *addrp, int type)
417 osi_assertx(addrp->sin_family == AF_INET, "unexpected socket value");
419 lock_ObtainWrite(&cm_serverLock);
420 for (tsp = cm_allServersp; tsp; tsp=tsp->allNextp) {
421 if (tsp->type == type &&
422 tsp->addr.sin_addr.s_addr == addrp->sin_addr.s_addr)
426 /* bump ref count if we found the server */
428 cm_GetServerNoLock(tsp);
430 /* drop big table lock */
431 lock_ReleaseWrite(&cm_serverLock);
433 /* return what we found */
437 cm_server_vols_t *cm_NewServerVols(void) {
438 cm_server_vols_t *tsvp;
440 tsvp = malloc(sizeof(*tsvp));
442 memset(tsvp, 0, sizeof(*tsvp));
447 cm_serverRef_t *cm_NewServerRef(cm_server_t *serverp, afs_uint32 volID)
449 cm_serverRef_t *tsrp;
450 cm_server_vols_t **tsrvpp = NULL;
451 afs_uint32 *slotp = NULL;
454 cm_GetServer(serverp);
455 tsrp = malloc(sizeof(*tsrp));
456 tsrp->server = serverp;
457 tsrp->status = srv_not_busy;
462 /* if we have a non-zero volID, we need to add it to the list
463 * of volumes maintained by the server. There are two phases:
464 * (1) see if the volID is already in the list and (2) insert
465 * it into the first empty slot if it is not.
468 lock_ObtainMutex(&serverp->mx);
470 tsrvpp = &serverp->vols;
474 for (i=0; i<NUM_SERVER_VOLS; i++) {
475 if ((*tsrvpp)->ids[i] == volID) {
478 } else if (!slotp && (*tsrvpp)->ids[i] == 0) {
479 slotp = &(*tsrvpp)->ids[i];
486 tsrvpp = &(*tsrvpp)->nextp;
493 /* if we didn't find an empty slot in a current
494 * page we must need a new page */
495 *tsrvpp = cm_NewServerVols();
497 (*tsrvpp)->ids[0] = volID;
501 lock_ReleaseMutex(&serverp->mx);
507 LONG_PTR cm_ChecksumServerList(cm_serverRef_t *serversp)
511 cm_serverRef_t *tsrp;
513 lock_ObtainWrite(&cm_serverLock);
514 for (tsrp = serversp; tsrp; tsrp=tsrp->next) {
519 sum ^= (LONG_PTR) tsrp->server;
522 lock_ReleaseWrite(&cm_serverLock);
527 ** Insert a server into the server list keeping the list sorted in
528 ** asending order of ipRank.
530 ** The refCount of the cm_serverRef_t is increased
532 void cm_InsertServerList(cm_serverRef_t** list, cm_serverRef_t* element)
534 cm_serverRef_t *current=*list;
535 unsigned short ipRank = element->server->ipRank;
537 lock_ObtainWrite(&cm_serverLock);
538 element->refCount++; /* increase refCount */
540 /* insertion into empty list or at the beginning of the list */
541 if ( !current || (current->server->ipRank > ipRank) )
543 element->next = *list;
545 lock_ReleaseWrite(&cm_serverLock);
549 while ( current->next ) /* find appropriate place to insert */
551 if ( current->next->server->ipRank > ipRank )
553 else current = current->next;
555 element->next = current->next;
556 current->next = element;
557 lock_ReleaseWrite(&cm_serverLock);
560 ** Re-sort the server list with the modified rank
561 ** returns 0 if element was changed successfully.
562 ** returns 1 if list remained unchanged.
564 long cm_ChangeRankServer(cm_serverRef_t** list, cm_server_t* server)
566 cm_serverRef_t **current=list;
567 cm_serverRef_t *element=0;
569 /* if there is max of one element in the list, nothing to sort */
570 if ( (!*current) || !((*current)->next) )
571 return 1; /* list unchanged: return success */
573 lock_ObtainWrite(&cm_serverLock);
574 /* if the server is on the list, delete it from list */
577 if ( (*current)->server == server)
579 element = (*current);
580 *current = (*current)->next; /* delete it */
583 current = & ( (*current)->next);
585 lock_ReleaseWrite(&cm_serverLock);
587 /* if this volume is not replicated on this server */
589 return 1; /* server is not on list */
591 /* re-insert deleted element into the list with modified rank*/
592 cm_InsertServerList(list, element);
594 /* reduce refCount which was increased by cm_InsertServerList */
595 lock_ObtainWrite(&cm_serverLock);
597 lock_ReleaseWrite(&cm_serverLock);
601 ** If there are more than one server on the list and the first n servers on
602 ** the list have the same rank( n>1), then randomise among the first n servers.
604 void cm_RandomizeServer(cm_serverRef_t** list)
607 cm_serverRef_t* tsrp = *list, *lastTsrp;
608 unsigned short lowestRank;
610 /* an empty list or a list with only one element */
611 if ( !tsrp || ! tsrp->next )
614 lock_ObtainWrite(&cm_serverLock);
616 /* count the number of servers with the lowest rank */
617 lowestRank = tsrp->server->ipRank;
618 for ( count=1, tsrp=tsrp->next; tsrp; tsrp=tsrp->next)
620 if ( tsrp->server->ipRank != lowestRank)
626 /* if there is only one server with the lowest rank, we are done */
628 lock_ReleaseWrite(&cm_serverLock);
632 picked = rand() % count;
634 lock_ReleaseWrite(&cm_serverLock);
639 while (--picked >= 0)
644 lastTsrp->next = tsrp->next; /* delete random element from list*/
645 tsrp->next = *list; /* insert element at the beginning of list */
647 lock_ReleaseWrite(&cm_serverLock);
650 /* call cm_FreeServer while holding a write lock on cm_serverLock */
651 void cm_FreeServer(cm_server_t* serverp)
653 cm_server_vols_t * tsrvp, *nextp;
655 cm_PutServerNoLock(serverp);
656 if (serverp->refCount == 0)
658 /* we need to check to ensure that all of the connections
659 * for this server have a 0 refCount; otherwise, they will
660 * not be garbage collected
662 cm_GCConnections(serverp); /* connsp */
664 if (!(serverp->flags & CM_SERVERFLAG_PREF_SET)) {
665 lock_FinalizeMutex(&serverp->mx);
666 if ( cm_allServersp == serverp )
667 cm_allServersp = serverp->allNextp;
671 for(tsp = cm_allServersp; tsp->allNextp; tsp=tsp->allNextp) {
672 if ( tsp->allNextp == serverp ) {
673 tsp->allNextp = serverp->allNextp;
679 /* free the volid list */
680 for ( tsrvp = serverp->vols; tsrvp; tsrvp = nextp) {
681 nextp = tsrvp->nextp;
690 void cm_RemoveVolumeFromServer(cm_server_t * serverp, afs_uint32 volID)
692 cm_server_vols_t * tsrvp;
698 for (tsrvp = serverp->vols; tsrvp; tsrvp = tsrvp->nextp) {
699 for (i=0; i<NUM_SERVER_VOLS; i++) {
700 if (tsrvp->ids[i] == volID) {
708 void cm_FreeServerList(cm_serverRef_t** list, afs_uint32 flags)
710 cm_serverRef_t **current = list;
711 cm_serverRef_t **nextp = 0;
712 cm_serverRef_t * next = 0;
714 lock_ObtainWrite(&cm_serverLock);
718 nextp = &(*current)->next;
719 if (--((*current)->refCount) == 0) {
722 if ((*current)->volID)
723 cm_RemoveVolumeFromServer((*current)->server, (*current)->volID);
724 cm_FreeServer((*current)->server);
728 if (flags & CM_FREESERVERLIST_DELETE) {
729 (*current)->status = srv_deleted;
730 if ((*current)->volID)
731 cm_RemoveVolumeFromServer((*current)->server, (*current)->volID);
737 lock_ReleaseWrite(&cm_serverLock);