ubik: Introduce new address lock

Introduce a new lock to protect:
	ubik_server->addr[]
	ubik_server->disk_rxcid
	ubik_server->vote_rxcid
	ubikSecClass
	ubikSecIndex

Globals are put into a new addr_data structure along with the lock.

Based on analysis and design work from Jeffrey Hutzelman.

Change-Id: I33ef58a36e87a3925b310052f83bb3c6dd60d785
Reviewed-on: http://gerrit.openafs.org/4157
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Jeffrey Altman <jaltman@openafs.org>
Reviewed-by: Derrick Brashear <shadow@dementia.org>

diff --git a/src/ubik/beacon.c b/src/ubik/beacon.c
index b3e69bf..ae482bb 100644 (file)
--- a/src/ubik/beacon.c
+++ b/src/ubik/beacon.c
@@ -50,6 +50,9 @@ static void * securityRock = NULL;
 afs_int32 ubikSecIndex;
 struct rx_securityClass *ubikSecClass;
 
+/* Values protected by the address lock */
+struct addr_data addr_globals;
+
 /* Values protected by the beacon lock */
 struct beacon_data beacon_globals;
 
@@ -172,22 +175,23 @@ ubeacon_InitServerList(afs_uint32 ame, afs_uint32 aservers[])
     return code;
 }
 
+/* Must be called with address lock held */
 void
 ubeacon_InitSecurityClass(void)
 {
     int i;
     /* get the security index to use, if we can */
     if (secLayerProc) {
-       i = (*secLayerProc) (securityRock, &ubikSecClass, &ubikSecIndex);
+       i = (*secLayerProc) (securityRock, &addr_globals.ubikSecClass, &addr_globals.ubikSecIndex);
     } else if (ubik_CRXSecurityProc) {
-       i = (*ubik_CRXSecurityProc) (ubik_CRXSecurityRock, &ubikSecClass,
-                                    &ubikSecIndex);
+       i = (*ubik_CRXSecurityProc) (ubik_CRXSecurityRock, &addr_globals.ubikSecClass,
+                                    &addr_globals.ubikSecIndex);
     } else
        i = 1;
     if (i) {
        /* don't have sec module yet */
-       ubikSecIndex = 0;
-       ubikSecClass = rxnull_NewClientSecurityObject();
+       addr_globals.ubikSecIndex = 0;
+       addr_globals.ubikSecClass = rxnull_NewClientSecurityObject();
     }
 }
 
@@ -198,11 +202,12 @@ ubeacon_ReinitServer(struct ubik_server *ts)
        struct rx_connection *disk_rxcid;
        struct rx_connection *vote_rxcid;
        struct rx_connection *tmp;
+       UBIK_ADDR_LOCK;
        ubeacon_InitSecurityClass();
        disk_rxcid =
            rx_NewConnection(rx_HostOf(rx_PeerOf(ts->disk_rxcid)),
                             ubik_callPortal, DISK_SERVICE_ID,
-                            ubikSecClass, ubikSecIndex);
+                            addr_globals.ubikSecClass, addr_globals.ubikSecIndex);
        if (disk_rxcid) {
            tmp = ts->disk_rxcid;
            ts->disk_rxcid = disk_rxcid;
@@ -211,12 +216,13 @@ ubeacon_ReinitServer(struct ubik_server *ts)
        vote_rxcid =
            rx_NewConnection(rx_HostOf(rx_PeerOf(ts->vote_rxcid)),
                             ubik_callPortal, VOTE_SERVICE_ID,
-                            ubikSecClass, ubikSecIndex);
+                            addr_globals.ubikSecClass, addr_globals.ubikSecIndex);
        if (vote_rxcid) {
            tmp = ts->vote_rxcid;
            ts->vote_rxcid = vote_rxcid;
            rx_PutConnection(tmp);
        }
+       UBIK_ADDR_UNLOCK;
     }
 }
 
@@ -300,12 +306,12 @@ ubeacon_InitServerListCommon(afs_uint32 ame, struct afsconf_cell *info,
            ts->vote_rxcid =
                rx_NewConnection(info->hostAddr[i].sin_addr.s_addr,
                                 ubik_callPortal, VOTE_SERVICE_ID,
-                                ubikSecClass, ubikSecIndex);
+                                addr_globals.ubikSecClass, addr_globals.ubikSecIndex);
            /* for disk reqs */
            ts->disk_rxcid =
                rx_NewConnection(info->hostAddr[i].sin_addr.s_addr,
                                 ubik_callPortal, DISK_SERVICE_ID,
-                                ubikSecClass, ubikSecIndex);
+                                addr_globals.ubikSecClass, addr_globals.ubikSecIndex);
            ts->up = 1;
        }
     } else {
@@ -318,8 +324,10 @@ ubeacon_InitServerListCommon(afs_uint32 ame, struct afsconf_cell *info,
            ts->next = ubik_servers;
            ubik_servers = ts;
            ts->addr[0] = servAddr;     /* primary address in  net byte order */
-           ts->vote_rxcid = rx_NewConnection(servAddr, ubik_callPortal, VOTE_SERVICE_ID, ubikSecClass, ubikSecIndex);  /* for vote reqs */
-           ts->disk_rxcid = rx_NewConnection(servAddr, ubik_callPortal, DISK_SERVICE_ID, ubikSecClass, ubikSecIndex);  /* for disk reqs */
+           ts->vote_rxcid = rx_NewConnection(servAddr, ubik_callPortal, VOTE_SERVICE_ID,
+                       addr_globals.ubikSecClass, addr_globals.ubikSecIndex);  /* for vote reqs */
+           ts->disk_rxcid = rx_NewConnection(servAddr, ubik_callPortal, DISK_SERVICE_ID,
+                       addr_globals.ubikSecClass, addr_globals.ubikSecIndex);  /* for disk reqs */
            ts->isClone = 0;    /* don't know about clones */
            ts->up = 1;
            if (ntohl((afs_uint32) servAddr) < (afs_uint32) magicHost) {
@@ -425,12 +433,14 @@ ubeacon_Interact(void *dummy)
         * prepare to send them an r multi-call containing the beacon message */
        i = 0;                  /* collect connections */
        UBIK_BEACON_LOCK;
+       UBIK_ADDR_LOCK;
        for (ts = ubik_servers; ts; ts = ts->next) {
            if (ts->up && ts->addr[0] != ubik_host[0]) {
                servers[i] = ts;
                connections[i++] = ts->vote_rxcid;
            }
        }
+       UBIK_ADDR_UNLOCK;
        UBIK_BEACON_UNLOCK;
        servers[i] = (struct ubik_server *)0;   /* end of list */
        /* note that we assume in the vote module that we'll always get at least BIGTIME
@@ -721,10 +731,12 @@ updateUbikNetworkAddress(afs_uint32 ubik_host[UBIK_MAX_INTERFACE_ADDR])
     char buffer[32];
     char hoststr[16];
 
+    UBIK_ADDR_LOCK;
     for (count = 0, ts = ubik_servers; ts; count++, ts = ts->next) {
        conns[count] = ts->disk_rxcid;
        server[count] = ts;
     }
+    UBIK_ADDR_UNLOCK;
 
 
     /* inform all other servers only if there are more than one
@@ -741,6 +753,7 @@ updateUbikNetworkAddress(afs_uint32 ubik_host[UBIK_MAX_INTERFACE_ADDR])
            multi_DISK_UpdateInterfaceAddr(&inAddr, &outAddr);
            ts = server[multi_i];       /* reply received from this server */
            if (!multi_error) {
+               UBIK_ADDR_LOCK;
                if (ts->addr[0] != htonl(outAddr.hostAddr[0])) {
                    code = UBADHOST;
                    strcpy(buffer, afs_inet_ntoa_r(ts->addr[0], hoststr));
@@ -751,15 +764,20 @@ updateUbikNetworkAddress(afs_uint32 ubik_host[UBIK_MAX_INTERFACE_ADDR])
                    for (j = 1; j < UBIK_MAX_INTERFACE_ADDR; j++)
                        ts->addr[j] = htonl(outAddr.hostAddr[j]);
                }
+               UBIK_ADDR_UNLOCK;
            } else if (multi_error == RXGEN_OPCODE) {   /* pre 3.5 remote server */
+               UBIK_ADDR_LOCK;
                ubik_print
                    ("ubik server %s does not support UpdateInterfaceAddr RPC\n",
                     afs_inet_ntoa_r(ts->addr[0], hoststr));
+               UBIK_ADDR_UNLOCK;
            } else if (multi_error == UBADHOST) {
                code = UBADHOST;        /* remote CellServDB inconsistency */
                ubik_print("Inconsistent Cell Info on server: ");
+               UBIK_ADDR_LOCK;
                for (j = 0; j < UBIK_MAX_INTERFACE_ADDR && ts->addr[j]; j++)
                    ubik_print("%s ", afs_inet_ntoa_r(ts->addr[j], hoststr));
+               UBIK_ADDR_UNLOCK;
                ubik_print("\n");
            } else {
                UBIK_BEACON_LOCK;

diff --git a/src/ubik/recovery.c b/src/ubik/recovery.c
index 79147f3..b3a0071 100644 (file)
--- a/src/ubik/recovery.c
+++ b/src/ubik/recovery.c
@@ -528,7 +528,9 @@ urecovery_Interact(void *dummy)
                UBIK_BEACON_UNLOCK;
                if (ts->isClone)
                    continue;
+               UBIK_ADDR_LOCK;
                code = DISK_GetVersion(ts->disk_rxcid, &ts->version);
+               UBIK_ADDR_UNLOCK;
                if (code == 0) {
                    /* perhaps this is the best version */
                    if (vcmp(ts->version, bestDBVersion) > 0) {
@@ -571,10 +573,12 @@ urecovery_Interact(void *dummy)
            /* Rx code to do the Bulk fetch */
            file = 0;
            offset = 0;
+           UBIK_ADDR_LOCK;
            rxcall = rx_NewCall(bestServer->disk_rxcid);
 
            ubik_print("Ubik: Synchronize database with server %s\n",
                       afs_inet_ntoa_r(bestServer->addr[0], hoststr));
+           UBIK_ADDR_UNLOCK;
 
            code = StartDISK_GetFile(rxcall, file);
            if (code) {
@@ -761,7 +765,9 @@ urecovery_Interact(void *dummy)
            }
 
            for (ts = ubik_servers; ts; ts = ts->next) {
+               UBIK_ADDR_LOCK;
                inAddr.s_addr = ts->addr[0];
+               UBIK_ADDR_UNLOCK;
                UBIK_BEACON_LOCK;
                if (!ts->up) {
                    UBIK_BEACON_UNLOCK;
@@ -781,7 +787,9 @@ urecovery_Interact(void *dummy)
                    if (!code) {
                        length = ubikstat.size;
                        file = offset = 0;
+                       UBIK_ADDR_LOCK;
                        rxcall = rx_NewCall(ts->disk_rxcid);
+                       UBIK_ADDR_UNLOCK;
                        code =
                            StartDISK_SendFile(rxcall, file, length,
                                               &ubik_dbase->version);
@@ -848,14 +856,13 @@ DoProbe(struct ubik_server *server)
     afs_uint32 addr;
     char buffer[32];
     char hoststr[16];
-    extern afs_int32 ubikSecIndex;
-    extern struct rx_securityClass *ubikSecClass;
 
+    UBIK_ADDR_LOCK;
     for (i = 0; (addr = server->addr[i]) && (i < UBIK_MAX_INTERFACE_ADDR);
         i++) {
        conns[i] =
            rx_NewConnection(addr, ubik_callPortal, DISK_SERVICE_ID,
-                            ubikSecClass, ubikSecIndex);
+                            addr_globals.ubikSecClass, addr_globals.ubikSecIndex);
 
        /* user requirement to use only the primary interface */
        if (ubikPrimaryAddrOnly) {
@@ -863,6 +870,7 @@ DoProbe(struct ubik_server *server)
            break;
        }
     }
+    UBIK_ADDR_UNLOCK;
     osi_Assert(i);                     /* at least one interface address for this server */
 
 #ifdef AFS_PTHREAD_ENV
@@ -883,6 +891,7 @@ DoProbe(struct ubik_server *server)
 #endif
 
     if (success_i >= 0) {
+       UBIK_ADDR_LOCK;
        addr = server->addr[success_i]; /* successful interface addr */
 
        if (server->disk_rxcid) /* destroy existing conn */
@@ -893,14 +902,15 @@ DoProbe(struct ubik_server *server)
        /* make new connections */
        server->disk_rxcid = conns[success_i];
        server->vote_rxcid = rx_NewConnection(addr, ubik_callPortal,
-                                             VOTE_SERVICE_ID, ubikSecClass,
-                                             ubikSecIndex);
+                                             VOTE_SERVICE_ID, addr_globals.ubikSecClass,
+                                             addr_globals.ubikSecIndex);
 
        connSuccess = conns[success_i];
        strcpy(buffer, afs_inet_ntoa_r(server->addr[0], hoststr));
 
        ubik_print("ubik:server %s is back up: will be contacted through %s\n",
             buffer, afs_inet_ntoa_r(addr, hoststr));
+       UBIK_ADDR_UNLOCK;
     }
 
     /* Destroy all connections except the one on which we succeeded */

diff --git a/src/ubik/remote.c b/src/ubik/remote.c
index 44b132f..7dd560c 100644 (file)
--- a/src/ubik/remote.c
+++ b/src/ubik/remote.c
@@ -605,6 +605,7 @@ SDISK_UpdateInterfaceAddr(struct rx_call *rxcall,
     int i, j, found = 0, probableMatch = 0;
     char hoststr[16];
 
+    UBIK_ADDR_LOCK;
     /* copy the output parameters */
     for (i = 0; i < UBIK_MAX_INTERFACE_ADDR; i++)
        outAddr->hostAddr[i] = ntohl(ubik_host[i]);
@@ -646,6 +647,7 @@ SDISK_UpdateInterfaceAddr(struct rx_call *rxcall,
        fflush(stdout);
        fflush(stderr);
        printServerInfo();
+       UBIK_ADDR_UNLOCK;
        return UBADHOST;
     }
 
@@ -658,6 +660,7 @@ SDISK_UpdateInterfaceAddr(struct rx_call *rxcall,
        ubik_print("%s ", afs_inet_ntoa_r(ts->addr[i], hoststr));
     ubik_print("\n");
 
+    UBIK_ADDR_UNLOCK;
     return 0;
 }
 

diff --git a/src/ubik/ubik.c b/src/ubik/ubik.c
index d3d0b99..c73e781 100644 (file)
--- a/src/ubik/ubik.c
+++ b/src/ubik/ubik.c
@@ -105,11 +105,15 @@ Quorum_StartIO(struct ubik_trans *atrans, struct ubik_server *as)
 {
     struct rx_connection *conn;
 
+    UBIK_ADDR_LOCK;
     conn = as->disk_rxcid;
 
 #ifdef AFS_PTHREAD_ENV
     rx_GetConnection(conn);
+    UBIK_ADDR_UNLOCK;
     DBRELE(atrans->dbase);
+#else
+    UBIK_ADDR_UNLOCK;
 #endif /* AFS_PTHREAD_ENV */
 
     return conn;
@@ -407,6 +411,7 @@ ubik_ServerInitCommon(afs_uint32 myHost, short myPort,
     MUTEX_INIT(&tdb->versionLock, "version lock", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&beacon_globals.beacon_lock, "beacon lock", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&vote_globals.vote_lock, "vote lock", MUTEX_DEFAULT, 0);
+    MUTEX_INIT(&addr_globals.addr_lock, "address lock", MUTEX_DEFAULT, 0);
 #else
     Lock_Init(&tdb->versionLock);
 #endif
@@ -1368,10 +1373,14 @@ ubikGetPrimaryInterfaceAddr(afs_uint32 addr)
     struct ubik_server *ts;
     int j;
 
+    UBIK_ADDR_LOCK;
     for (ts = ubik_servers; ts; ts = ts->next)
        for (j = 0; j < UBIK_MAX_INTERFACE_ADDR; j++)
-           if (ts->addr[j] == addr)
+           if (ts->addr[j] == addr) {
+               UBIK_ADDR_UNLOCK;
                return ts->addr[0];     /* net byte order */
+           }
+    UBIK_ADDR_UNLOCK;
     return 0;                  /* if not in server database, return error */
 }
 

diff --git a/src/ubik/ubik.p.h b/src/ubik/ubik.p.h
index 20aebc7..7851896 100644 (file)
--- a/src/ubik/ubik.p.h
+++ b/src/ubik/ubik.p.h
@@ -393,6 +393,24 @@ struct vote_data {
 #define UBIK_VOTE_LOCK MUTEX_ENTER(&vote_globals.vote_lock)
 #define UBIK_VOTE_UNLOCK MUTEX_EXIT(&vote_globals.vote_lock)
 
+/*!
+ * \brief Server address data.  All values are protected by addr_lock
+ *
+ * This lock also protects:
+ *     ubik_server: addr[], vote_rxcid, disk_rxcid
+ *
+ */
+struct addr_data {
+#ifdef AFS_PTHREAD_ENV
+    pthread_mutex_t addr_lock;
+#endif
+    afs_int32 ubikSecIndex;
+    struct rx_securityClass *ubikSecClass;
+};
+
+#define UBIK_ADDR_LOCK MUTEX_ENTER(&addr_globals.addr_lock)
+#define UBIK_ADDR_UNLOCK MUTEX_EXIT(&addr_globals.addr_lock)
+
 /* phys.c */
 extern int uphys_stat(struct ubik_dbase *adbase, afs_int32 afid,
                      struct ubik_stat *astat);
@@ -477,6 +495,7 @@ extern int ubeacon_InitServerListByInfo(afs_uint32 ame,
 extern int ubeacon_InitServerList(afs_uint32 ame, afs_uint32 aservers[]);
 extern void *ubeacon_Interact(void *);
 extern struct beacon_data beacon_globals;
+extern struct addr_data addr_globals;
 
 /*\}*/