2 * Copyright 2000, International Business Machines Corporation and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
10 #include <afsconfig.h>
11 #include <afs/param.h>
24 #include <WINNT\syscfg.h>
25 #include <WINNT/afsreg.h>
30 osi_rwlock_t cm_serverLock;
31 osi_rwlock_t cm_syscfgLock;
33 cm_server_t *cm_allServersp;
34 afs_uint32 cm_numFileServers = 0;
35 afs_uint32 cm_numVldbServers = 0;
38 cm_ForceNewConnectionsAllServers(void)
42 lock_ObtainRead(&cm_serverLock);
43 for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
44 cm_GetServerNoLock(tsp);
45 lock_ReleaseRead(&cm_serverLock);
46 cm_ForceNewConnections(tsp);
47 lock_ObtainRead(&cm_serverLock);
48 cm_PutServerNoLock(tsp);
50 lock_ReleaseRead(&cm_serverLock);
54 cm_ServerClearRPCStats(void) {
58 lock_ObtainRead(&cm_serverLock);
59 for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
63 rx_ClearPeerRPCStats(opcode_VL_ProbeServer>>32, tsp->addr.sin_addr.s_addr, port);
67 rx_ClearPeerRPCStats(opcode_RXAFS_GetCapabilities>>32, tsp->addr.sin_addr.s_addr, port);
68 rx_ClearPeerRPCStats(opcode_RXAFS_GetTime>>32, tsp->addr.sin_addr.s_addr, port);
72 lock_ReleaseRead(&cm_serverLock);
76 * lock_ObtainMutex must be held prior to calling
80 cm_RankServer(cm_server_t * tsp)
82 afs_int32 code = 0; /* start with "success" */
83 struct rx_debugPeer tpeer;
84 struct rx_peer * rxPeer;
87 afs_uint64 perfRank = 0;
91 int isDown = (tsp->flags & CM_SERVERFLAG_DOWN);
92 void *peerRpcStats = NULL;
93 afs_uint64 opcode = 0;
98 opcode = opcode_VL_ProbeServer;
102 opcode = opcode_RXAFS_GetCapabilities;
108 cm_SetServerIPRank(tsp);
114 * There are three potential components to the ranking:
115 * 1. Any administrative set preference whether it be
116 * via "fs setserverprefs", registry or dns.
118 * 2. Network subnet mask comparison.
120 * 3. Performance data.
122 * If there is an administrative rank, that is the
123 * the primary factor. If not the primary factor
124 * is the network ranking.
127 code = rx_GetLocalPeers(tsp->addr.sin_addr.s_addr, port, &tpeer);
129 peerRpcStats = rx_CopyPeerRPCStats(opcode, tsp->addr.sin_addr.s_addr, port);
130 if (peerRpcStats == NULL && tsp->type == CM_SERVER_FILE)
131 peerRpcStats = rx_CopyPeerRPCStats(opcode_RXAFS_GetTime, tsp->addr.sin_addr.s_addr, port);
133 afs_uint64 execTimeSum = _8THMSEC(RPCOpStat_ExecTimeSum(peerRpcStats));
134 afs_uint64 queueTimeSum = _8THMSEC(RPCOpStat_QTimeSum(peerRpcStats));
135 afs_uint64 numCalls = RPCOpStat_NumCalls(peerRpcStats);
138 rtt = (execTimeSum - queueTimeSum) / numCalls;
140 rx_ReleaseRPCStats(peerRpcStats);
143 if (rtt == 0 && tpeer.rtt) {
150 perfRank += (6000 * log_rtt / 5000) * 5000;
152 if (tsp->type == CM_SERVER_FILE) {
153 /* give an edge to servers with high congestion windows */
154 perfRank -= (tpeer.cwind - 1)* 15;
159 if (tsp->adminRank) {
160 newRank = tsp->adminRank * 0.8;
161 newRank += tsp->ipRank * 0.2;
163 newRank = tsp->ipRank;
167 newRank += perfRank * 0.1;
169 newRank += (rand() & 0x000f); /* randomize */
171 if (newRank > 0xFFFF)
172 osi_Log1(afsd_logp, "new server rank %I64u exceeds 0xFFFF", newRank);
175 * If the ranking changes by more than the randomization
176 * factor, update the server reference lists.
178 if (abs(newRank - tsp->activeRank) > 0xf) {
179 tsp->activeRank = newRank;
181 lock_ReleaseMutex(&tsp->mx);
185 * find volumes which might have RO copy
186 * on server and change the ordering of
189 cm_ChangeRankVolume(tsp);
192 /* set preferences for an existing vlserver */
193 cm_ChangeRankCellVLServer(tsp);
196 lock_ObtainMutex(&tsp->mx);
204 cm_PingServer(cm_server_t *tsp)
209 struct rx_connection * rxconnp;
210 Capabilities caps = {0, 0};
214 lock_ObtainMutex(&tsp->mx);
215 if (tsp->flags & CM_SERVERFLAG_PINGING) {
217 osi_SleepM((LONG_PTR)tsp, &tsp->mx);
218 lock_ObtainMutex(&tsp->mx);
220 if (tsp->waitCount == 0)
221 _InterlockedAnd(&tsp->flags, ~CM_SERVERFLAG_PINGING);
223 osi_Wakeup((LONG_PTR)tsp);
224 lock_ReleaseMutex(&tsp->mx);
227 _InterlockedOr(&tsp->flags, CM_SERVERFLAG_PINGING);
228 wasDown = tsp->flags & CM_SERVERFLAG_DOWN;
229 afs_inet_ntoa_r(tsp->addr.sin_addr.S_un.S_addr, hoststr);
230 lock_ReleaseMutex(&tsp->mx);
232 code = cm_ConnByServer(tsp, cm_rootUserp, FALSE, &connp);
234 /* now call the appropriate ping call. Drop the timeout if
235 * the server is known to be down, so that we don't waste a
236 * lot of time retiming out down servers.
239 osi_Log4(afsd_logp, "cm_PingServer server %s (%s) was %s with caps 0x%x",
240 osi_LogSaveString(afsd_logp, hoststr),
241 tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
242 wasDown ? "down" : "up",
245 rxconnp = cm_GetRxConn(connp);
247 rx_SetConnDeadTime(rxconnp, 10);
248 if (tsp->type == CM_SERVER_VLDB) {
249 code = VL_ProbeServer(rxconnp);
253 code = RXAFS_GetCapabilities(rxconnp, &caps);
256 rx_SetConnDeadTime(rxconnp, ConnDeadtimeout);
257 rx_PutConnection(rxconnp);
259 } /* got an unauthenticated connection to this server */
261 lock_ObtainMutex(&tsp->mx);
262 if (code >= 0 || code == RXGEN_OPCODE || code == RX_CALL_BUSY) {
263 /* mark server as up */
264 _InterlockedAnd(&tsp->flags, ~CM_SERVERFLAG_DOWN);
267 /* we currently handle 32-bits of capabilities */
268 if (code != RXGEN_OPCODE && code != RX_CALL_BUSY &&
269 caps.Capabilities_len > 0) {
270 tsp->capabilities = caps.Capabilities_val[0];
271 xdr_free((xdrproc_t) xdr_Capabilities, &caps);
272 caps.Capabilities_len = 0;
273 caps.Capabilities_val = 0;
275 tsp->capabilities = 0;
278 osi_Log3(afsd_logp, "cm_PingServer server %s (%s) is up with caps 0x%x",
279 osi_LogSaveString(afsd_logp, hoststr),
280 tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
283 /* Now update the volume status if necessary */
285 cm_server_vols_t * tsrvp;
289 for (tsrvp = tsp->vols; tsrvp; tsrvp = tsrvp->nextp) {
290 for (i=0; i<NUM_SERVER_VOLS; i++) {
291 if (tsrvp->ids[i] != 0) {
294 lock_ReleaseMutex(&tsp->mx);
295 code = cm_FindVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
296 &req, CM_GETVOL_FLAG_NO_LRU_UPDATE, &volp);
297 lock_ObtainMutex(&tsp->mx);
299 cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
308 /* mark server as down */
309 if (!(tsp->flags & CM_SERVERFLAG_DOWN)) {
310 _InterlockedOr(&tsp->flags, CM_SERVERFLAG_DOWN);
311 tsp->downTime = time(NULL);
313 if (code != VRESTARTING) {
314 lock_ReleaseMutex(&tsp->mx);
315 cm_ForceNewConnections(tsp);
316 lock_ObtainMutex(&tsp->mx);
318 osi_Log3(afsd_logp, "cm_PingServer server %s (%s) is down with caps 0x%x",
319 osi_LogSaveString(afsd_logp, hoststr),
320 tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
323 /* Now update the volume status if necessary */
325 cm_server_vols_t * tsrvp;
329 for (tsrvp = tsp->vols; tsrvp; tsrvp = tsrvp->nextp) {
330 for (i=0; i<NUM_SERVER_VOLS; i++) {
331 if (tsrvp->ids[i] != 0) {
334 lock_ReleaseMutex(&tsp->mx);
335 code = cm_FindVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
336 &req, CM_GETVOL_FLAG_NO_LRU_UPDATE, &volp);
337 lock_ObtainMutex(&tsp->mx);
339 cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
349 if (tsp->waitCount == 0)
350 _InterlockedAnd(&tsp->flags, ~CM_SERVERFLAG_PINGING);
352 osi_Wakeup((LONG_PTR)tsp);
353 lock_ReleaseMutex(&tsp->mx);
361 lock_ObtainRead(&cm_serverLock);
362 for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
363 cm_GetServerNoLock(tsp);
364 lock_ReleaseRead(&cm_serverLock);
366 lock_ObtainMutex(&tsp->mx);
368 /* if the server is not down, rank the server */
369 if(!(tsp->flags & CM_SERVERFLAG_DOWN))
372 lock_ReleaseMutex(&tsp->mx);
374 lock_ObtainRead(&cm_serverLock);
375 cm_PutServerNoLock(tsp);
377 lock_ReleaseRead(&cm_serverLock);
380 static void cm_CheckServersSingular(afs_uint32 flags, cm_cell_t *cellp)
382 /* ping all file servers, up or down, with unauthenticated connection,
383 * to find out whether we have all our callbacks from the server still.
384 * Also, ping down VLDBs.
392 lock_ObtainRead(&cm_serverLock);
393 for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
394 cm_GetServerNoLock(tsp);
395 lock_ReleaseRead(&cm_serverLock);
397 /* now process the server */
398 lock_ObtainMutex(&tsp->mx);
401 isDown = tsp->flags & CM_SERVERFLAG_DOWN;
402 isFS = tsp->type == CM_SERVER_FILE;
403 isVLDB = tsp->type == CM_SERVER_VLDB;
405 /* only do the ping if the cell matches the requested cell, or we're
406 * matching all cells (cellp == NULL), and if we've requested to ping
407 * this type of {up, down} servers.
409 if ((cellp == NULL || cellp == tsp->cellp) &&
410 ((isDown && (flags & CM_FLAG_CHECKDOWNSERVERS)) ||
411 (!isDown && (flags & CM_FLAG_CHECKUPSERVERS))) &&
412 ((!(flags & CM_FLAG_CHECKVLDBSERVERS) ||
413 isVLDB && (flags & CM_FLAG_CHECKVLDBSERVERS)) &&
414 (!(flags & CM_FLAG_CHECKFILESERVERS) ||
415 isFS && (flags & CM_FLAG_CHECKFILESERVERS)))) {
417 } /* we're supposed to check this up/down server */
418 lock_ReleaseMutex(&tsp->mx);
420 /* at this point, we've adjusted the server state, so do the ping and
426 /* also, run the GC function for connections on all of the
427 * server's connections.
429 cm_GCConnections(tsp);
431 lock_ObtainRead(&cm_serverLock);
432 cm_PutServerNoLock(tsp);
434 lock_ReleaseRead(&cm_serverLock);
437 static void cm_CheckServersMulti(afs_uint32 flags, cm_cell_t *cellp)
440 * The goal of this function is to probe simultaneously
441 * probe all of the up/down servers (vldb/file) as
442 * specified by flags in the minimum number of RPCs.
443 * Effectively that means use one multi_RXAFS_GetCapabilities()
444 * followed by possibly one multi_RXAFS_GetTime() and
445 * one multi_VL_ProbeServer().
447 * To make this work we must construct the list of vldb
448 * and file servers that are to be probed as well as the
449 * associated data structures.
452 int srvAddrCount = 0;
453 struct srvAddr **addrs = NULL;
454 cm_conn_t **conns = NULL;
455 struct rx_connection **rxconns = NULL;
457 afs_int32 i, nconns = 0, maxconns;
458 afs_int32 *conntimer, *results;
459 Capabilities *caps = NULL;
460 cm_server_t ** serversp, *tsp;
461 afs_uint32 isDown, wasDown;
467 maxconns = max(cm_numFileServers,cm_numVldbServers);
471 conns = (cm_conn_t **)malloc(maxconns * sizeof(cm_conn_t *));
472 rxconns = (struct rx_connection **)malloc(maxconns * sizeof(struct rx_connection *));
473 conntimer = (afs_int32 *)malloc(maxconns * sizeof (afs_int32));
474 results = (afs_int32 *)malloc(maxconns * sizeof (afs_int32));
475 serversp = (cm_server_t **)malloc(maxconns * sizeof(cm_server_t *));
476 caps = (Capabilities *)malloc(maxconns * sizeof(Capabilities));
478 memset(caps, 0, maxconns * sizeof(Capabilities));
480 if ((flags & CM_FLAG_CHECKFILESERVERS) ||
481 !(flags & (CM_FLAG_CHECKFILESERVERS|CM_FLAG_CHECKVLDBSERVERS)))
483 lock_ObtainRead(&cm_serverLock);
484 for (nconns=0, tsp = cm_allServersp; tsp && nconns < maxconns; tsp = tsp->allNextp) {
485 if (tsp->type != CM_SERVER_FILE ||
486 tsp->cellp == NULL || /* SetPref only */
487 cellp && cellp != tsp->cellp)
490 cm_GetServerNoLock(tsp);
491 lock_ReleaseRead(&cm_serverLock);
493 lock_ObtainMutex(&tsp->mx);
494 isDown = tsp->flags & CM_SERVERFLAG_DOWN;
496 if ((tsp->flags & CM_SERVERFLAG_PINGING) ||
497 !((isDown && (flags & CM_FLAG_CHECKDOWNSERVERS)) ||
498 (!isDown && (flags & CM_FLAG_CHECKUPSERVERS)))) {
499 lock_ReleaseMutex(&tsp->mx);
500 lock_ObtainRead(&cm_serverLock);
501 cm_PutServerNoLock(tsp);
505 _InterlockedOr(&tsp->flags, CM_SERVERFLAG_PINGING);
506 lock_ReleaseMutex(&tsp->mx);
508 serversp[nconns] = tsp;
509 code = cm_ConnByServer(tsp, cm_rootUserp, FALSE, &conns[nconns]);
511 lock_ObtainRead(&cm_serverLock);
512 cm_PutServerNoLock(tsp);
515 lock_ObtainRead(&cm_serverLock);
516 rxconns[nconns] = cm_GetRxConn(conns[nconns]);
517 if (conntimer[nconns] = (isDown ? 1 : 0))
518 rx_SetConnDeadTime(rxconns[nconns], 10);
522 lock_ReleaseRead(&cm_serverLock);
525 /* Perform the multi call */
527 multi_Rx(rxconns,nconns)
529 multi_RXAFS_GetCapabilities(&caps[multi_i]);
530 results[multi_i]=multi_error;
534 /* Process results of servers that support RXAFS_GetCapabilities */
535 for (i=0; i<nconns; i++) {
537 rx_SetConnDeadTime(rxconns[i], ConnDeadtimeout);
538 rx_PutConnection(rxconns[i]);
539 cm_PutConn(conns[i]);
542 cm_GCConnections(tsp);
544 lock_ObtainMutex(&tsp->mx);
545 wasDown = tsp->flags & CM_SERVERFLAG_DOWN;
547 if (results[i] >= 0 || results[i] == RXGEN_OPCODE ||
548 results[i] == RX_CALL_BUSY) {
549 /* mark server as up */
550 _InterlockedAnd(&tsp->flags, ~CM_SERVERFLAG_DOWN);
553 /* we currently handle 32-bits of capabilities */
554 if (results[i] != RXGEN_OPCODE && results[i] != RX_CALL_BUSY &&
555 caps[i].Capabilities_len > 0) {
556 tsp->capabilities = caps[i].Capabilities_val[0];
557 xdr_free((xdrproc_t) xdr_Capabilities, &caps[i]);
558 caps[i].Capabilities_len = 0;
559 caps[i].Capabilities_val = 0;
561 tsp->capabilities = 0;
564 afs_inet_ntoa_r(tsp->addr.sin_addr.S_un.S_addr, hoststr);
565 osi_Log3(afsd_logp, "cm_MultiPingServer server %s (%s) is up with caps 0x%x",
566 osi_LogSaveString(afsd_logp, hoststr),
567 tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
570 /* Now update the volume status if necessary */
572 cm_server_vols_t * tsrvp;
576 for (tsrvp = tsp->vols; tsrvp; tsrvp = tsrvp->nextp) {
577 for (i=0; i<NUM_SERVER_VOLS; i++) {
578 if (tsrvp->ids[i] != 0) {
581 lock_ReleaseMutex(&tsp->mx);
582 code = cm_FindVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
583 &req, CM_GETVOL_FLAG_NO_LRU_UPDATE, &volp);
584 lock_ObtainMutex(&tsp->mx);
586 cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
595 /* mark server as down */
596 if (!(tsp->flags & CM_SERVERFLAG_DOWN)) {
597 _InterlockedOr(&tsp->flags, CM_SERVERFLAG_DOWN);
598 tsp->downTime = time(NULL);
600 if (code != VRESTARTING) {
601 lock_ReleaseMutex(&tsp->mx);
602 cm_ForceNewConnections(tsp);
603 lock_ObtainMutex(&tsp->mx);
605 afs_inet_ntoa_r(tsp->addr.sin_addr.S_un.S_addr, hoststr);
606 osi_Log3(afsd_logp, "cm_MultiPingServer server %s (%s) is down with caps 0x%x",
607 osi_LogSaveString(afsd_logp, hoststr),
608 tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
611 /* Now update the volume status if necessary */
613 cm_server_vols_t * tsrvp;
617 for (tsrvp = tsp->vols; tsrvp; tsrvp = tsrvp->nextp) {
618 for (i=0; i<NUM_SERVER_VOLS; i++) {
619 if (tsrvp->ids[i] != 0) {
622 lock_ReleaseMutex(&tsp->mx);
623 code = cm_FindVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
624 &req, CM_GETVOL_FLAG_NO_LRU_UPDATE, &volp);
625 lock_ObtainMutex(&tsp->mx);
627 cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
637 if (tsp->waitCount == 0)
638 _InterlockedAnd(&tsp->flags, ~CM_SERVERFLAG_PINGING);
640 osi_Wakeup((LONG_PTR)tsp);
642 lock_ReleaseMutex(&tsp->mx);
648 if ((flags & CM_FLAG_CHECKVLDBSERVERS) ||
649 !(flags & (CM_FLAG_CHECKFILESERVERS|CM_FLAG_CHECKVLDBSERVERS)))
651 lock_ObtainRead(&cm_serverLock);
652 for (nconns=0, tsp = cm_allServersp; tsp && nconns < maxconns; tsp = tsp->allNextp) {
653 if (tsp->type != CM_SERVER_VLDB ||
654 tsp->cellp == NULL || /* SetPref only */
655 cellp && cellp != tsp->cellp)
658 cm_GetServerNoLock(tsp);
659 lock_ReleaseRead(&cm_serverLock);
661 lock_ObtainMutex(&tsp->mx);
662 isDown = tsp->flags & CM_SERVERFLAG_DOWN;
664 if ((tsp->flags & CM_SERVERFLAG_PINGING) ||
665 !((isDown && (flags & CM_FLAG_CHECKDOWNSERVERS)) ||
666 (!isDown && (flags & CM_FLAG_CHECKUPSERVERS)))) {
667 lock_ReleaseMutex(&tsp->mx);
668 lock_ObtainRead(&cm_serverLock);
669 cm_PutServerNoLock(tsp);
673 _InterlockedOr(&tsp->flags, CM_SERVERFLAG_PINGING);
674 lock_ReleaseMutex(&tsp->mx);
676 serversp[nconns] = tsp;
677 code = cm_ConnByServer(tsp, cm_rootUserp, FALSE, &conns[nconns]);
679 lock_ObtainRead(&cm_serverLock);
680 cm_PutServerNoLock(tsp);
683 lock_ObtainRead(&cm_serverLock);
684 rxconns[nconns] = cm_GetRxConn(conns[nconns]);
685 conntimer[nconns] = (isDown ? 1 : 0);
687 rx_SetConnDeadTime(rxconns[nconns], 10);
691 lock_ReleaseRead(&cm_serverLock);
694 /* Perform the multi call */
696 multi_Rx(rxconns,nconns)
698 multi_VL_ProbeServer();
699 results[multi_i]=multi_error;
703 /* Process results of servers that support VL_ProbeServer */
704 for (i=0; i<nconns; i++) {
706 rx_SetConnDeadTime(rxconns[i], ConnDeadtimeout);
707 rx_PutConnection(rxconns[i]);
708 cm_PutConn(conns[i]);
711 cm_GCConnections(tsp);
713 lock_ObtainMutex(&tsp->mx);
714 wasDown = tsp->flags & CM_SERVERFLAG_DOWN;
716 if (results[i] >= 0 || results[i] == RX_CALL_BUSY) {
717 /* mark server as up */
718 _InterlockedAnd(&tsp->flags, ~CM_SERVERFLAG_DOWN);
720 tsp->capabilities = 0;
722 afs_inet_ntoa_r(tsp->addr.sin_addr.S_un.S_addr, hoststr);
723 osi_Log3(afsd_logp, "cm_MultiPingServer server %s (%s) is up with caps 0x%x",
724 osi_LogSaveString(afsd_logp, hoststr),
725 tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
730 /* mark server as down */
731 if (!(tsp->flags & CM_SERVERFLAG_DOWN)) {
732 _InterlockedOr(&tsp->flags, CM_SERVERFLAG_DOWN);
733 tsp->downTime = time(NULL);
735 if (code != VRESTARTING) {
736 lock_ReleaseMutex(&tsp->mx);
737 cm_ForceNewConnections(tsp);
738 lock_ObtainMutex(&tsp->mx);
740 afs_inet_ntoa_r(tsp->addr.sin_addr.S_un.S_addr, hoststr);
741 osi_Log3(afsd_logp, "cm_MultiPingServer server %s (%s) is down with caps 0x%x",
742 osi_LogSaveString(afsd_logp, hoststr),
743 tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
749 if (tsp->waitCount == 0)
750 _InterlockedAnd(&tsp->flags, ~CM_SERVERFLAG_PINGING);
752 osi_Wakeup((LONG_PTR)tsp);
754 lock_ReleaseMutex(&tsp->mx);
768 void cm_CheckServers(afs_uint32 flags, cm_cell_t *cellp)
775 code = RegOpenKeyEx(HKEY_LOCAL_MACHINE, AFSREG_CLT_SVC_PARAM_SUBKEY,
776 0, KEY_QUERY_VALUE, &parmKey);
777 if (code == ERROR_SUCCESS) {
778 dummyLen = sizeof(multi);
779 code = RegQueryValueEx(parmKey, "MultiCheckServers", NULL, NULL,
780 (BYTE *) &multi, &dummyLen);
781 RegCloseKey (parmKey);
785 cm_CheckServersMulti(flags, cellp);
787 cm_CheckServersSingular(flags, cellp);
790 void cm_InitServer(void)
792 static osi_once_t once;
794 if (osi_Once(&once)) {
795 lock_InitializeRWLock(&cm_serverLock, "cm_serverLock", LOCK_HIERARCHY_SERVER_GLOBAL);
796 lock_InitializeRWLock(&cm_syscfgLock, "cm_syscfgLock", LOCK_HIERARCHY_SYSCFG_GLOBAL);
801 /* Protected by cm_syscfgLock (rw) */
802 int cm_noIPAddr; /* number of client network interfaces */
803 int cm_IPAddr[CM_MAXINTERFACE_ADDR]; /* client's IP address in host order */
804 int cm_SubnetMask[CM_MAXINTERFACE_ADDR];/* client's subnet mask in host order*/
805 int cm_NetMtu[CM_MAXINTERFACE_ADDR]; /* client's MTU sizes */
806 int cm_NetFlags[CM_MAXINTERFACE_ADDR]; /* network flags */
807 int cm_LanAdapterChangeDetected = 1;
809 void cm_SetLanAdapterChangeDetected(void)
811 lock_ObtainWrite(&cm_syscfgLock);
812 cm_LanAdapterChangeDetected = 1;
813 lock_ReleaseWrite(&cm_syscfgLock);
816 void cm_GetServer(cm_server_t *serverp)
818 lock_ObtainRead(&cm_serverLock);
819 InterlockedIncrement(&serverp->refCount);
820 lock_ReleaseRead(&cm_serverLock);
823 void cm_GetServerNoLock(cm_server_t *serverp)
825 InterlockedIncrement(&serverp->refCount);
828 void cm_PutServer(cm_server_t *serverp)
831 lock_ObtainRead(&cm_serverLock);
832 refCount = InterlockedDecrement(&serverp->refCount);
833 osi_assertx(refCount >= 0, "cm_server_t refCount underflow");
834 lock_ReleaseRead(&cm_serverLock);
837 void cm_PutServerNoLock(cm_server_t *serverp)
839 afs_int32 refCount = InterlockedDecrement(&serverp->refCount);
840 osi_assertx(refCount >= 0, "cm_server_t refCount underflow");
843 void cm_SetServerNo64Bit(cm_server_t * serverp, int no64bit)
845 lock_ObtainMutex(&serverp->mx);
847 _InterlockedOr(&serverp->flags, CM_SERVERFLAG_NO64BIT);
849 _InterlockedAnd(&serverp->flags, ~CM_SERVERFLAG_NO64BIT);
850 lock_ReleaseMutex(&serverp->mx);
853 void cm_SetServerNoInlineBulk(cm_server_t * serverp, int no)
855 lock_ObtainMutex(&serverp->mx);
857 _InterlockedOr(&serverp->flags, CM_SERVERFLAG_NOINLINEBULK);
859 _InterlockedAnd(&serverp->flags, ~CM_SERVERFLAG_NOINLINEBULK);
860 lock_ReleaseMutex(&serverp->mx);
863 void cm_SetServerIPRank(cm_server_t * serverp)
865 unsigned long serverAddr; /* in host byte order */
866 unsigned long myAddr, myNet, mySubnet;/* in host byte order */
867 unsigned long netMask;
871 lock_ObtainRead(&cm_syscfgLock);
872 if (cm_LanAdapterChangeDetected) {
873 lock_ConvertRToW(&cm_syscfgLock);
874 if (cm_LanAdapterChangeDetected) {
875 /* get network related info */
876 cm_noIPAddr = CM_MAXINTERFACE_ADDR;
877 code = syscfg_GetIFInfo(&cm_noIPAddr,
878 cm_IPAddr, cm_SubnetMask,
879 cm_NetMtu, cm_NetFlags);
880 cm_LanAdapterChangeDetected = 0;
882 lock_ConvertWToR(&cm_syscfgLock);
885 serverAddr = ntohl(serverp->addr.sin_addr.s_addr);
886 serverp->ipRank = CM_IPRANK_LOW; /* default settings */
888 for ( i=0; i < cm_noIPAddr; i++)
890 /* loop through all the client's IP address and compare
891 ** each of them against the server's IP address */
893 myAddr = cm_IPAddr[i];
894 if ( IN_CLASSA(myAddr) )
895 netMask = IN_CLASSA_NET;
896 else if ( IN_CLASSB(myAddr) )
897 netMask = IN_CLASSB_NET;
898 else if ( IN_CLASSC(myAddr) )
899 netMask = IN_CLASSC_NET;
903 myNet = myAddr & netMask;
904 mySubnet = myAddr & cm_SubnetMask[i];
906 if ( (serverAddr & netMask) == myNet )
908 if ( (serverAddr & cm_SubnetMask[i]) == mySubnet)
910 if ( serverAddr == myAddr ) {
911 serverp->ipRank = min(serverp->ipRank,
912 CM_IPRANK_TOP);/* same machine */
914 serverp->ipRank = min(serverp->ipRank,
915 CM_IPRANK_HI); /* same subnet */
918 serverp->ipRank = min(serverp->ipRank, CM_IPRANK_MED); /* same net */
921 } /* and of for loop */
922 lock_ReleaseRead(&cm_syscfgLock);
925 cm_server_t *cm_NewServer(struct sockaddr_in *socketp, int type, cm_cell_t *cellp, afsUUID *uuidp, afs_uint32 flags) {
928 osi_assertx(socketp->sin_family == AF_INET, "unexpected socket family");
930 lock_ObtainWrite(&cm_serverLock); /* get server lock */
931 tsp = cm_FindServer(socketp, type, TRUE);
933 /* we might have found a server created by set server prefs */
934 if (uuidp && !afs_uuid_is_nil(uuidp) &&
935 !(tsp->flags & CM_SERVERFLAG_UUID))
938 _InterlockedOr(&tsp->flags, CM_SERVERFLAG_UUID);
940 lock_ReleaseWrite(&cm_serverLock);
944 tsp = malloc(sizeof(*tsp));
946 memset(tsp, 0, sizeof(*tsp));
949 if (uuidp && !afs_uuid_is_nil(uuidp)) {
951 _InterlockedOr(&tsp->flags, CM_SERVERFLAG_UUID);
954 lock_InitializeMutex(&tsp->mx, "cm_server_t mutex", LOCK_HIERARCHY_SERVER);
955 tsp->addr = *socketp;
957 tsp->allNextp = cm_allServersp;
958 cm_allServersp = tsp;
969 lock_ReleaseWrite(&cm_serverLock); /* release server lock */
972 if (!(flags & CM_FLAG_NOPROBE)) {
973 _InterlockedOr(&tsp->flags, CM_SERVERFLAG_DOWN); /* assume down; ping will mark up if available */
974 lock_ObtainMutex(&tsp->mx);
976 lock_ReleaseMutex(&tsp->mx);
977 cm_PingServer(tsp); /* Obtain Capabilities and check up/down state */
980 pthread_attr_t tattr;
983 /* Probe the server in the background to determine if it is up or down */
984 pthread_attr_init(&tattr);
985 pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED);
987 lock_ObtainMutex(&tsp->mx);
989 lock_ReleaseMutex(&tsp->mx);
990 pstatus = pthread_create(&phandle, &tattr, cm_PingServer, tsp);
992 pthread_attr_destroy(&tattr);
999 cm_FindServerByIP(afs_uint32 ipaddr, unsigned short port, int type, int locked)
1004 lock_ObtainRead(&cm_serverLock);
1006 for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
1007 if (tsp->type == type &&
1008 tsp->addr.sin_addr.S_un.S_addr == ipaddr &&
1009 (tsp->addr.sin_port == port || tsp->addr.sin_port == 0))
1013 /* bump ref count if we found the server */
1015 cm_GetServerNoLock(tsp);
1018 lock_ReleaseRead(&cm_serverLock);
1024 cm_FindServerByUuid(afsUUID *serverUuid, int type, int locked)
1029 lock_ObtainRead(&cm_serverLock);
1031 for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
1032 if (tsp->type == type && !afs_uuid_equal(&tsp->uuid, serverUuid))
1036 /* bump ref count if we found the server */
1038 cm_GetServerNoLock(tsp);
1041 lock_ReleaseRead(&cm_serverLock);
1046 /* find a server based on its properties */
1047 cm_server_t *cm_FindServer(struct sockaddr_in *addrp, int type, int locked)
1049 osi_assertx(addrp->sin_family == AF_INET, "unexpected socket value");
1051 return cm_FindServerByIP(addrp->sin_addr.s_addr, addrp->sin_port, type, locked);
1054 cm_server_vols_t *cm_NewServerVols(void) {
1055 cm_server_vols_t *tsvp;
1057 tsvp = malloc(sizeof(*tsvp));
1059 memset(tsvp, 0, sizeof(*tsvp));
1065 * cm_NewServerRef() returns with the allocated cm_serverRef_t
1066 * with a refCount of 1.
1068 cm_serverRef_t *cm_NewServerRef(cm_server_t *serverp, afs_uint32 volID)
1070 cm_serverRef_t *tsrp;
1071 cm_server_vols_t **tsrvpp = NULL;
1072 afs_uint32 *slotp = NULL;
1075 cm_GetServer(serverp);
1076 tsrp = malloc(sizeof(*tsrp));
1077 tsrp->server = serverp;
1078 tsrp->status = srv_not_busy;
1080 tsrp->volID = volID;
1083 /* if we have a non-zero volID, we need to add it to the list
1084 * of volumes maintained by the server. There are two phases:
1085 * (1) see if the volID is already in the list and (2) insert
1086 * it into the first empty slot if it is not.
1089 lock_ObtainMutex(&serverp->mx);
1091 tsrvpp = &serverp->vols;
1095 for (i=0; i<NUM_SERVER_VOLS; i++) {
1096 if ((*tsrvpp)->ids[i] == volID) {
1099 } else if (!slotp && (*tsrvpp)->ids[i] == 0) {
1100 slotp = &(*tsrvpp)->ids[i];
1107 tsrvpp = &(*tsrvpp)->nextp;
1114 /* if we didn't find an empty slot in a current
1115 * page we must need a new page */
1116 *tsrvpp = cm_NewServerVols();
1118 (*tsrvpp)->ids[0] = volID;
1122 lock_ReleaseMutex(&serverp->mx);
1128 void cm_GetServerRef(cm_serverRef_t *tsrp, int locked)
1133 lock_ObtainRead(&cm_serverLock);
1134 refCount = InterlockedIncrement(&tsrp->refCount);
1136 lock_ReleaseRead(&cm_serverLock);
1139 afs_int32 cm_PutServerRef(cm_serverRef_t *tsrp, int locked)
1144 lock_ObtainRead(&cm_serverLock);
1145 refCount = InterlockedDecrement(&tsrp->refCount);
1146 osi_assertx(refCount >= 0, "cm_serverRef_t refCount underflow");
1149 lock_ReleaseRead(&cm_serverLock);
1155 cm_ServerListSize(cm_serverRef_t* serversp)
1157 afs_uint32 count = 0;
1158 cm_serverRef_t *tsrp;
1160 lock_ObtainRead(&cm_serverLock);
1161 for (tsrp = serversp; tsrp; tsrp=tsrp->next) {
1162 if (tsrp->status == srv_deleted)
1166 lock_ReleaseRead(&cm_serverLock);
1170 LONG_PTR cm_ChecksumServerList(cm_serverRef_t *serversp)
1174 cm_serverRef_t *tsrp;
1176 lock_ObtainRead(&cm_serverLock);
1177 for (tsrp = serversp; tsrp; tsrp=tsrp->next) {
1178 if (tsrp->status == srv_deleted)
1184 sum ^= (LONG_PTR) tsrp->server;
1187 lock_ReleaseRead(&cm_serverLock);
1192 ** Insert a server into the server list keeping the list sorted in
1193 ** ascending order of ipRank.
1195 ** The refCount of the cm_serverRef_t is not altered.
1197 void cm_InsertServerList(cm_serverRef_t** list, cm_serverRef_t* element)
1199 cm_serverRef_t *current;
1200 unsigned short rank;
1202 lock_ObtainWrite(&cm_serverLock);
1204 * Since we are grabbing the serverLock exclusively remove any
1205 * deleted serverRef objects with a zero refcount before
1206 * inserting the new item.
1209 cm_serverRef_t **currentp = list;
1210 cm_serverRef_t **nextp = NULL;
1211 cm_serverRef_t * next = NULL;
1212 cm_server_t * serverp = NULL;
1214 for (currentp = list; *currentp; currentp = nextp)
1216 nextp = &(*currentp)->next;
1217 /* obtain a refcnt on next in case cm_serverLock is dropped */
1219 cm_GetServerRef(*nextp, TRUE);
1220 if ((*currentp)->refCount == 0 &&
1221 (*currentp)->status == srv_deleted) {
1224 if ((*currentp)->volID)
1225 cm_RemoveVolumeFromServer((*currentp)->server, (*currentp)->volID);
1226 serverp = (*currentp)->server;
1229 /* cm_FreeServer will drop cm_serverLock if serverp->refCount == 0 */
1230 cm_FreeServer(serverp);
1232 /* drop the next refcnt obtained above. */
1234 cm_PutServerRef(*nextp, TRUE);
1238 /* insertion into empty list or at the beginning of the list */
1241 element->next = NULL;
1247 * Now that deleted entries have been removed and we know that the
1248 * list was not empty, look for duplicates. If the element we are
1249 * inserting already exists, discard it.
1251 for ( current = *list; current; current = current->next)
1253 cm_server_t * server1 = current->server;
1254 cm_server_t * server2 = element->server;
1256 if (current->status == srv_deleted)
1259 if (server1->type != server2->type)
1262 if (server1->addr.sin_addr.s_addr != server2->addr.sin_addr.s_addr)
1265 if ((server1->flags & CM_SERVERFLAG_UUID) != (server2->flags & CM_SERVERFLAG_UUID))
1268 if ((server1->flags & CM_SERVERFLAG_UUID) &&
1269 !afs_uuid_equal(&server1->uuid, &server2->uuid))
1272 /* we must have a match, discard the new element */
1277 rank = element->server->activeRank;
1279 /* insertion at the beginning of the list */
1280 if ((*list)->server->activeRank > rank)
1282 element->next = *list;
1287 /* find appropriate place to insert */
1288 for ( current = *list; current->next; current = current->next)
1290 if ( current->next->server->activeRank > rank )
1293 element->next = current->next;
1294 current->next = element;
1297 lock_ReleaseWrite(&cm_serverLock);
1300 ** Re-sort the server list with the modified rank
1301 ** returns 0 if element was changed successfully.
1302 ** returns 1 if list remained unchanged.
1304 long cm_ChangeRankServer(cm_serverRef_t** list, cm_server_t* server)
1306 cm_serverRef_t **current;
1307 cm_serverRef_t *element;
1309 lock_ObtainWrite(&cm_serverLock);
1313 /* if there is max of one element in the list, nothing to sort */
1314 if ( (!*current) || !((*current)->next) ) {
1315 lock_ReleaseWrite(&cm_serverLock);
1316 return 1; /* list unchanged: return success */
1319 /* if the server is on the list, delete it from list */
1322 if ( (*current)->server == server)
1324 element = (*current);
1325 *current = element->next; /* delete it */
1328 current = & ( (*current)->next);
1330 lock_ReleaseWrite(&cm_serverLock);
1332 /* if this volume is not replicated on this server */
1334 return 1; /* server is not on list */
1336 /* re-insert deleted element into the list with modified rank*/
1337 cm_InsertServerList(list, element);
1342 ** If there are more than one server on the list and the first n servers on
1343 ** the list have the same rank( n>1), then randomise among the first n servers.
1345 void cm_RandomizeServer(cm_serverRef_t** list)
1348 cm_serverRef_t* tsrp, *lastTsrp;
1349 unsigned short lowestRank;
1351 lock_ObtainWrite(&cm_serverLock);
1354 /* an empty list or a list with only one element */
1355 if ( !tsrp || ! tsrp->next ) {
1356 lock_ReleaseWrite(&cm_serverLock);
1360 /* count the number of servers with the lowest rank */
1361 lowestRank = tsrp->server->activeRank;
1362 for ( count=1, tsrp=tsrp->next; tsrp; tsrp=tsrp->next)
1364 if ( tsrp->server->activeRank != lowestRank)
1370 /* if there is only one server with the lowest rank, we are done */
1372 lock_ReleaseWrite(&cm_serverLock);
1376 picked = rand() % count;
1378 lock_ReleaseWrite(&cm_serverLock);
1383 while (--picked >= 0)
1388 lastTsrp->next = tsrp->next; /* delete random element from list*/
1389 tsrp->next = *list; /* insert element at the beginning of list */
1391 lock_ReleaseWrite(&cm_serverLock);
1394 /* call cm_FreeServer while holding a write lock on cm_serverLock */
1395 void cm_FreeServer(cm_server_t* serverp)
1397 cm_server_vols_t * tsrvp, *nextp;
1400 cm_PutServerNoLock(serverp);
1401 if (serverp->refCount == 0)
1404 * we need to check to ensure that all of the connections
1405 * for this server have a 0 refCount; otherwise, they will
1406 * not be garbage collected
1408 * must drop the cm_serverLock because cm_GCConnections
1409 * obtains the cm_connLock and that comes first in the
1412 lock_ReleaseWrite(&cm_serverLock);
1413 cm_GCConnections(serverp); /* connsp */
1414 lock_ObtainWrite(&cm_serverLock);
1419 * Once we have the cm_serverLock locked check to make
1420 * sure the refCount is still zero before removing the
1423 if (serverp->refCount == 0) {
1424 if (!(serverp->flags & CM_SERVERFLAG_PREF_SET)) {
1425 switch (serverp->type) {
1426 case CM_SERVER_VLDB:
1427 cm_numVldbServers--;
1429 case CM_SERVER_FILE:
1430 cm_numFileServers--;
1434 lock_FinalizeMutex(&serverp->mx);
1435 if ( cm_allServersp == serverp )
1436 cm_allServersp = serverp->allNextp;
1440 for(tsp = cm_allServersp; tsp->allNextp; tsp=tsp->allNextp) {
1441 if ( tsp->allNextp == serverp ) {
1442 tsp->allNextp = serverp->allNextp;
1448 /* free the volid list */
1449 for ( tsrvp = serverp->vols; tsrvp; tsrvp = nextp) {
1450 nextp = tsrvp->nextp;
1459 /* Called with cm_serverLock write locked */
1460 void cm_RemoveVolumeFromServer(cm_server_t * serverp, afs_uint32 volID)
1462 cm_server_vols_t * tsrvp;
1468 for (tsrvp = serverp->vols; tsrvp; tsrvp = tsrvp->nextp) {
1469 for (i=0; i<NUM_SERVER_VOLS; i++) {
1470 if (tsrvp->ids[i] == volID) {
1478 int cm_IsServerListEmpty(cm_serverRef_t *serversp)
1480 cm_serverRef_t *tsrp;
1483 if (serversp == NULL)
1484 return CM_ERROR_EMPTY;
1486 lock_ObtainRead(&cm_serverLock);
1487 for (tsrp = serversp; tsrp; tsrp=tsrp->next) {
1488 if (tsrp->status == srv_deleted)
1493 lock_ReleaseRead(&cm_serverLock);
1495 return ( allDeleted ? CM_ERROR_EMPTY : 0 );
1498 void cm_FreeServerList(cm_serverRef_t** list, afs_uint32 flags)
1500 cm_serverRef_t **current;
1501 cm_serverRef_t **nextp;
1502 cm_serverRef_t * next;
1503 cm_server_t * serverp;
1506 lock_ObtainWrite(&cm_serverLock);
1516 nextp = &(*current)->next;
1517 /* obtain a refcnt on next in case cm_serverLock is dropped */
1519 cm_GetServerRef(*nextp, TRUE);
1520 refCount = cm_PutServerRef(*current, TRUE);
1521 if (refCount == 0) {
1524 if ((*current)->volID)
1525 cm_RemoveVolumeFromServer((*current)->server, (*current)->volID);
1526 serverp = (*current)->server;
1529 /* cm_FreeServer will drop cm_serverLock if serverp->refCount == 0 */
1530 cm_FreeServer(serverp);
1532 if (flags & CM_FREESERVERLIST_DELETE) {
1533 (*current)->status = srv_deleted;
1534 if ((*current)->volID)
1535 cm_RemoveVolumeFromServer((*current)->server, (*current)->volID);
1539 /* drop the next refcnt obtained above. */
1541 cm_PutServerRef(*current, TRUE);
1546 lock_ReleaseWrite(&cm_serverLock);
1549 /* dump all servers to a file.
1550 * cookie is used to identify this batch for easy parsing,
1551 * and it a string provided by a caller
1553 int cm_DumpServers(FILE *outputFile, char *cookie, int lock)
1562 lock_ObtainRead(&cm_serverLock);
1565 "%s - dumping servers - cm_numFileServers=%d, cm_numVldbServers=%d\r\n",
1566 cookie, cm_numFileServers, cm_numVldbServers);
1567 WriteFile(outputFile, output, (DWORD)strlen(output), &zilch, NULL);
1569 for (tsp = cm_allServersp; tsp; tsp=tsp->allNextp)
1574 switch (tsp->type) {
1575 case CM_SERVER_VLDB:
1578 case CM_SERVER_FILE:
1585 afsUUID_to_string(&tsp->uuid, uuidstr, sizeof(uuidstr));
1586 afs_inet_ntoa_r(tsp->addr.sin_addr.s_addr, hoststr);
1587 down = ctime(&tsp->downTime);
1588 down[strlen(down)-1] = '\0';
1591 "%s - tsp=0x%p cell=%s addr=%-15s port=%u uuid=%s type=%s caps=0x%x "
1592 "flags=0x%x waitCount=%u rank=%u downTime=\"%s\" refCount=%u\r\n",
1593 cookie, tsp, tsp->cellp ? tsp->cellp->name : "", hoststr,
1594 ntohs(tsp->addr.sin_port), uuidstr, type,
1595 tsp->capabilities, tsp->flags, tsp->waitCount, tsp->activeRank,
1596 (tsp->flags & CM_SERVERFLAG_DOWN) ? "down" : "up",
1598 WriteFile(outputFile, output, (DWORD)strlen(output), &zilch, NULL);
1600 sprintf(output, "%s - Done dumping servers.\r\n", cookie);
1601 WriteFile(outputFile, output, (DWORD)strlen(output), &zilch, NULL);
1604 lock_ReleaseRead(&cm_serverLock);
1610 * Determine if two servers are in fact the same.
1612 * Returns 1 if they match, 0 if they do not
1614 int cm_ServerEqual(cm_server_t *srv1, cm_server_t *srv2)
1618 if (srv1 == NULL || srv2 == NULL)
1624 if (srv1->flags & CM_SERVERFLAG_UUID) {
1625 if (!(srv2->flags & CM_SERVERFLAG_UUID))
1628 /* Both support UUID */
1629 if (UuidEqual((UUID *)&srv1->uuid, (UUID *)&srv2->uuid, &status))
1632 if (srv1->flags & CM_SERVERFLAG_UUID)
1635 /* Neither support UUID so perform an addr/port comparison */
1636 if ( srv1->addr.sin_family == srv2->addr.sin_family &&
1637 srv1->addr.sin_addr.s_addr == srv2->addr.sin_addr.s_addr &&
1638 srv1->addr.sin_port == srv2->addr.sin_port )