viced-host-uuid-and-addr-hashing-corrections-20090530
[openafs.git] / src / viced / host.c
index 747a747..7e72394 100644 (file)
@@ -903,12 +903,10 @@ h_LookupUuid_r(afsUUID * uuidp)
        assert(host);
        if (!(host->hostFlags & HOSTDELETED) && host->interface
            && afs_uuid_equal(&host->interface->uuid, uuidp)) {
-           break;
+            return host;
        }
-       host = NULL;
     }
-    return host;
-
+    return NULL;
 }                              /*h_Lookup */
 
 
@@ -939,7 +937,7 @@ h_TossStuff_r(register struct host *host)
     if (h_NBLock_r(host) != 0) {
        char hoststr[16];
        ViceLog(0,
-               ("Warning:  h_TossStuff_r failed; Host %x (%s:%d) was locked.\n",
+               ("Warning:  h_TossStuff_r failed; Host %" AFS_PTR_FMT " (%s:%d) was locked.\n",
                 host, afs_inet_ntoa_r(host->host, hoststr), ntohs(host->port)));
        return;
     } else {
@@ -954,7 +952,7 @@ h_TossStuff_r(register struct host *host)
            if (code < 0) {
                char hoststr[16];
                ViceLog(0,
-                       ("Warning: h_TossStuff_r failed: Host %x (%s:%d) client %x was locked.\n",
+                       ("Warning: h_TossStuff_r failed: Host %" AFS_PTR_FMT " (%s:%d) client %x was locked.\n",
                         host, afs_inet_ntoa_r(host->host, hoststr),
                         ntohs(host->port), client));
                return;
@@ -963,7 +961,7 @@ h_TossStuff_r(register struct host *host)
            if (client->refCount) {
                char hoststr[16];
                ViceLog(0,
-                       ("Warning: h_TossStuff_r failed: Host %x (%s:%d) client %x refcount %d.\n",
+                       ("Warning: h_TossStuff_r failed: Host %" AFS_PTR_FMT " (%s:%d) client %x refcount %d.\n",
                         host, afs_inet_ntoa_r(host->host, hoststr),
                         ntohs(host->port), client, client->refCount));
                /* This is the same thing we do if the host is locked */
@@ -986,9 +984,7 @@ h_TossStuff_r(register struct host *host)
     host->hostFlags &= ~CLIENTDELETED;
 
     if (host->hostFlags & HOSTDELETED) {
-        register struct h_AddrHashChain **ahp, *ath;
        register struct rx_connection *rxconn;
-       afsUUID *uuidp;
        struct AddrPort hostAddrPort;
        int i;
 
@@ -1007,44 +1003,22 @@ h_TossStuff_r(register struct host *host)
 
        /* if alternate addresses do not exist */
        if (!(host->interface)) {
-           for (ahp = &hostAddrHashTable[h_HashIndex(host->host)]; (ath = *ahp);
-                ahp = &ath->next) {
-               assert(ath->hostPtr);
-               if (ath->hostPtr == host) {
-                   *ahp = ath->next;
-                   free(ath);
-                   break;
-               }
-           }
+           h_DeleteHostFromAddrHashTable_r(host->host, host->port, host);
        } else {
-            register struct h_UuidHashChain **uhp, *uth;
-           /* delete the hash entry for the UUID */
-           uuidp = &host->interface->uuid;
-           for (uhp = &hostUuidHashTable[h_UuidHashIndex(uuidp)]; (uth = *uhp);
-                uhp = &uth->next) {
-               assert(uth->hostPtr);
-               if (uth->hostPtr == host) {
-                   *uhp = uth->next;
-                   free(uth);
-                   break;
-               }
-           }
-           /* delete the hash entry for each alternate addresses */
+            h_DeleteHostFromUuidHashTable_r(host);
+           h_DeleteHostFromAddrHashTable_r(host->host, host->port, host);
+           /* delete the hash entry for each valid alternate addresses */
            for (i = 0; i < host->interface->numberOfInterfaces; i++) {
                hostAddrPort = host->interface->interface[i];
-
-                if (!hostAddrPort.valid)
-                    continue;
-
-               for (ahp = &hostAddrHashTable[h_HashIndex(hostAddrPort.addr)]; (ath = *ahp);
-                    ahp = &ath->next) {
-                   assert(ath->hostPtr);
-                   if (ath->hostPtr == host) {
-                       *ahp = ath->next;
-                       free(ath);
-                       break;
-                   }
-               }
+                /* 
+                 * if the interface addr/port is the primary, we already
+                 * removed it.  If the addr/port is not valid, its not
+                 * in the hash table.
+                 */
+                if (hostAddrPort.valid &&
+                    (host->host != hostAddrPort.addr ||
+                     host->port != hostAddrPort.port))
+                    h_DeleteHostFromAddrHashTable_r(hostAddrPort.addr, hostAddrPort.port, host);
            }
            free(host->interface);
            host->interface = NULL;
@@ -1151,15 +1125,30 @@ h_AddHostToUuidHashTable_r(struct afsUUID *uuid, struct host *host)
 {
     int index;
     struct h_UuidHashChain *chain;
+    char uuid1[128], uuid2[128];
+    char hoststr[16];
 
     /* hash into proper bucket */
     index = h_UuidHashIndex(uuid);
 
     /* don't add the same entry multiple times */
     for (chain = hostUuidHashTable[index]; chain; chain = chain->next) {
+       if (!chain->hostPtr)
+           continue;
+
        if (chain->hostPtr->interface && 
-           afs_uuid_equal(&chain->hostPtr->interface->uuid, uuid))
+           afs_uuid_equal(&chain->hostPtr->interface->uuid, uuid)) {
+           if (LogLevel >= 125) {
+               afsUUID_to_string(&chain->hostPtr->interface->uuid, uuid1, 
+                                 127);
+               afsUUID_to_string(uuid, uuid2, 127);
+               ViceLog(125, ("h_AddHostToUuidHashTable_r: host %" AFS_PTR_FMT " (uuid %s) exists as %s:%d (uuid %s)\n", 
+                             host, uuid1,
+                             afs_inet_ntoa_r(chain->hostPtr->host, hoststr), 
+                             ntohs(chain->hostPtr->port), uuid2));
+           }
            return;
+       }
     }
 
     /* insert into beginning of list for this bucket */
@@ -1168,12 +1157,53 @@ h_AddHostToUuidHashTable_r(struct afsUUID *uuid, struct host *host)
        ViceLog(0, ("Failed malloc in h_AddHostToUuidHashTable_r\n"));
        assert(0);
     }
-    assert(chain);
     chain->hostPtr = host;
     chain->next = hostUuidHashTable[index];
     hostUuidHashTable[index] = chain;
+         if (LogLevel < 125)
+              return;
+     afsUUID_to_string(uuid, uuid2, 127);
+     ViceLog(125, 
+            ("h_AddHostToUuidHashTable_r: host %" AFS_PTR_FMT " (%s:%d) added as uuid %s\n",
+             host, afs_inet_ntoa_r(chain->hostPtr->host, hoststr), 
+             ntohs(chain->hostPtr->port), uuid));
 }
 
+/* deletes a HashChain structure corresponding to this host */
+int
+h_DeleteHostFromUuidHashTable_r(struct host *host)
+{
+     int index;
+     register struct h_UuidHashChain **uhp, *uth;
+     char uuid1[128];
+     char hoststr[16];
+     if (!host->interface)
+       return 0;
+     /* hash into proper bucket */
+     index = h_UuidHashIndex(&host->interface->uuid);
+     
+     if (LogLevel >= 125)
+        afsUUID_to_string(&host->interface->uuid, uuid1, 127);
+     for (uhp = &hostUuidHashTable[index]; (uth = *uhp); uhp = &uth->next) {
+         assert(uth->hostPtr);
+        if (uth->hostPtr == host) {
+            ViceLog(125, 
+                    ("h_DeleteHostFromUuidHashTable_r: host %" AFS_PTR_FMT " (uuid %s %s:%d)\n",
+                     host, uuid1, afs_inet_ntoa_r(host->host, hoststr), 
+                     ntohs(host->port)));
+            *uhp = uth->next;
+            free(uth);
+            return 1;
+        }
+     }
+     ViceLog(125, 
+            ("h_DeleteHostFromUuidHashTable_r: host %" AFS_PTR_FMT " (uuid %s %s:%d) not found\n",
+             host, uuid1, afs_inet_ntoa_r(host->host, hoststr), 
+             ntohs(host->port)));
+     return 0;
+}
 
 /* inserts a new HashChain structure corresponding to this address */
 void
@@ -1182,26 +1212,30 @@ h_AddHostToAddrHashTable_r(afs_uint32 addr, afs_uint16 port, struct host *host)
     int index;
     struct h_AddrHashChain *chain;
     int found = 0;
-    char hoststr[16];
+    char hoststr[16], hoststr2[16];
 
     /* hash into proper bucket */
     index = h_HashIndex(addr);
 
     /* don't add the same entry multiple times */
     for (chain = hostAddrHashTable[index]; chain; chain = chain->next) {
-       if (chain->addr == addr && chain->port == port) {
-            if (chain->hostPtr == host)
-                found = 1;
-            else if (!(host->hostFlags & HOSTDELETED))
-                ViceLog(125, ("Addr %s:%d assigned to %x and %x.\n",
-                            afs_inet_ntoa_r(addr, hoststr), ntohs(port),
-                            host, chain));
-        }
+       if (chain->hostPtr == host) {
+           if (chain->addr != addr || chain->port != port) {
+               ViceLog(0, 
+                       ("h_AddHostToAddrHashTable_r: host %" AFS_PTR_FMT " exists as %s:%d when adding %s:%d\n",
+                        host, afs_inet_ntoa_r(chain->addr, hoststr), 
+                        ntohs(chain->port), afs_inet_ntoa_r(addr, hoststr2), 
+                        ntohs(port)));
+           } else
+               ViceLog(125, 
+                       ("h_AddHostToAddrHashTable_r: host %" AFS_PTR_FMT " (%s:%d) already hashed\n",
+                        host, afs_inet_ntoa_r(chain->addr, hoststr), 
+                        ntohs(chain->port)));
+           
+           return;
+       }
     }
 
-    if (found)
-        return;
-
     /* insert into beginning of list for this bucket */
     chain = (struct h_AddrHashChain *)malloc(sizeof(struct h_AddrHashChain));
     if (!chain) {
@@ -1213,13 +1247,14 @@ h_AddHostToAddrHashTable_r(afs_uint32 addr, afs_uint16 port, struct host *host)
     chain->addr = addr;
     chain->port = port;
     hostAddrHashTable[index] = chain;
+    ViceLog(125, ("h_AddHostToAddrHashTable_r: host %" AFS_PTR_FMT " added as %s:%d\n",
+                 host, afs_inet_ntoa_r(addr, hoststr), ntohs(port)));
 }
 
 /*
- * This is called with host locked and held. At this point, the
- * hostAddrHashTable should not have entries for the alternate
- * interfaces. This function has to insert these entries in the
- * hostAddrHashTable.
+ * This is called with host locked and held. 
+ * It is called to either validate or add an additional interface
+ * address/port on the specified host.  
  *
  * All addresses are in network byte order.
  */
@@ -1228,10 +1263,9 @@ addInterfaceAddr_r(struct host *host, afs_uint32 addr, afs_uint16 port)
 {
     int i;
     int number;
-    int found;
     struct Interface *interface;
     char hoststr[16], hoststr2[16];
-
+                                                   
     assert(host);
     assert(host->interface);
 
@@ -1240,45 +1274,54 @@ addInterfaceAddr_r(struct host *host, afs_uint32 addr, afs_uint16 port)
      * for this host.
      */
     number = host->interface->numberOfInterfaces;
-    for (i = 0, found = 0; i < number && !found; i++) {
+    for (i = 0; i < number; i++) {
        if (host->interface->interface[i].addr == addr &&
              host->interface->interface[i].port == port) {
-           found = 1;
-            host->interface->interface[i].valid = 1;
+           ViceLog(125, 
+                   ("addInterfaceAddr : found host %" AFS_PTR_FMT " (%s:%d) adding %s:%d%s\n",
+                    host, afs_inet_ntoa_r(host->host, hoststr), 
+                    ntohs(host->port), afs_inet_ntoa_r(addr, hoststr2), 
+                    ntohs(port), host->interface->interface[i].valid ? "" : 
+                    ", validating"));
+     
+           if (host->interface->interface[i].valid == 0) {
+               host->interface->interface[i].valid = 1;
+               h_AddHostToAddrHashTable_r(addr, port, host);
+           }
+           return 0;
         }
     }
 
-    ViceLog(125, ("addInterfaceAddr : host %s:%d addr %s:%d : found:%d\n", 
-                  afs_inet_ntoa_r(host->host, hoststr), ntohs(host->port), 
-                  afs_inet_ntoa_r(addr, hoststr2), ntohs(port), found));
+    ViceLog(125, ("addInterfaceAddr : host %" AFS_PTR_FMT " (%s:%d) adding %s:%d\n", 
+                 host, afs_inet_ntoa_r(host->host, hoststr), 
+                 ntohs(host->port), afs_inet_ntoa_r(addr, hoststr2), 
+                 ntohs(port)));
     
-    if (!found) {
-       interface = (struct Interface *)
-           malloc(sizeof(struct Interface) + (sizeof(struct AddrPort) * number));
-       if (!interface) {
-           ViceLog(0, ("Failed malloc in addInterfaceAddr_r\n"));
-           assert(0);
-       }
-       interface->numberOfInterfaces = number + 1;
-       interface->uuid = host->interface->uuid;
-       for (i = 0; i < number; i++)
-           interface->interface[i] = host->interface->interface[i];
-       interface->interface[number].addr = addr;
-       interface->interface[number].port = port;
-        interface->interface[number].valid = 1;
-       free(host->interface);
-       host->interface = interface;
+    interface = (struct Interface *)
+       malloc(sizeof(struct Interface) + (sizeof(struct AddrPort) * number));
+    if (!interface) {
+       ViceLog(0, ("Failed malloc in addInterfaceAddr_r\n"));
+       assert(0);
     }
-
+    interface->numberOfInterfaces = number + 1;
+    interface->uuid = host->interface->uuid;
+    for (i = 0; i < number; i++)
+       interface->interface[i] = host->interface->interface[i];
+    
+    /* Add the new valid interface */
+    interface->interface[number].addr = addr;
+    interface->interface[number].port = port;
+    interface->interface[number].valid = 1;
+    h_AddHostToAddrHashTable_r(addr, port, host);
+    free(host->interface);
+    host->interface = interface;
+    
     return 0;
 }
 
 
 /*
- * This is called with host locked and held. At this point, the
- * hostAddrHashTable should not be having entries for the alternate
- * interfaces. This function has to insert these entries in the
- * hostAddrHashTable.
+ * This is called with host locked and held.
  *
  * All addresses are in network byte order.
  */
@@ -1287,16 +1330,16 @@ removeInterfaceAddr_r(struct host *host, afs_uint32 addr, afs_uint16 port)
 {
     int i;
     int number;
-    int found;
     struct Interface *interface;
     char hoststr[16], hoststr2[16];
 
     assert(host);
     assert(host->interface);
 
-    ViceLog(125, ("removeInterfaceAddr : host %s:%d addr %s:%d\n", 
-                  afs_inet_ntoa_r(host->host, hoststr), ntohs(host->port), 
-                  afs_inet_ntoa_r(addr, hoststr2), ntohs(port)));
+    ViceLog(125, ("removeInterfaceAddr : host %" AFS_PTR_FMT " (%s:%d) addr %s:%d\n", 
+                 host, afs_inet_ntoa_r(host->host, hoststr), 
+                 ntohs(host->port), afs_inet_ntoa_r(addr, hoststr2), 
+                 ntohs(port)));
 
     /*
      * Make sure this address is on the list of known addresses
@@ -1304,26 +1347,62 @@ removeInterfaceAddr_r(struct host *host, afs_uint32 addr, afs_uint16 port)
      */
     interface = host->interface;
     number = host->interface->numberOfInterfaces;
-    for (i = 0, found = 0; i < number && !found; i++) {
+    for (i = 0; i < number; i++) {
        if (interface->interface[i].addr == addr &&
            interface->interface[i].port == port) {
-           found = 1;
-            interface->interface[i].valid = 0;
-       }
-    }
-    if (found) {
-       number--;
-       for (; i < number; i++) {
-           interface->interface[i] = interface->interface[i+1];
+           if (interface->interface[i].valid)
+               h_DeleteHostFromAddrHashTable_r(addr, port, host);
+           number--;
+           for (; i < number; i++) {
+               interface->interface[i] = interface->interface[i+1];
+           }
+           interface->numberOfInterfaces = number;
+           return 0;
        }
-       interface->numberOfInterfaces = number;
-    }
+    }  
+    /* not found */
+    return 0;
+}
 
+/*
+ * This is called with host locked and held.
+ *
+ * All addresses are in network byte order.
+ */
+int
+invalidateInterfaceAddr_r(struct host *host, afs_uint32 addr, afs_uint16 port)
+{
+    int i;
+    int number;
+    struct Interface *interface;
+    char hoststr[16], hoststr2[16];
+    
+    assert(host);
+    assert(host->interface);
+    
+    ViceLog(125, ("invalidateInterfaceAddr : host %" AFS_PTR_FMT " (%s:%d) addr %s:%d\n", 
+                 host, afs_inet_ntoa_r(host->host, hoststr), 
+                 ntohs(host->port), afs_inet_ntoa_r(addr, hoststr2), 
+                 ntohs(port)));
+    
     /*
-     * Remove the hash table entry for this address
+     * Make sure this address is on the list of known addresses
+     * for this host.
      */
-    h_DeleteHostFromAddrHashTable_r(addr, port, host);
-
+    interface = host->interface;
+    number = host->interface->numberOfInterfaces;
+    for (i = 0; i < number; i++) {
+       if (interface->interface[i].addr == addr &&
+           interface->interface[i].port == port) {
+            if (interface->interface[i].valid) {
+                h_DeleteHostFromAddrHashTable_r(addr, port, host);
+               interface->interface[i].valid = 0;
+           }
+           return 0;
+       }
+    }
+    
+    /* not found */
     return 0;
 }
 
@@ -1341,23 +1420,29 @@ removeAddress_r(struct host *host, afs_uint32 addr, afs_uint16 port)
 {
     int i;
     char hoststr[16], hoststr2[16];
+    struct rx_connection *rxconn;
 
-    if (!host->interface) {
+    if (!host->interface || host->interface->numberOfInterfaces == 1) {
         if (host->host == addr && host->port == port) {
             ViceLog(25,
-                    ("Removing only address for host %x (%s:%d), deleting host.\n",
+                    ("Removing only address for host %" AFS_PTR_FMT " (%s:%d), deleting host.\n",
                      host, afs_inet_ntoa_r(host->host, hoststr), ntohs(host->port)));
             host->hostFlags |= HOSTDELETED;
+            /* 
+             * Do not remove the primary addr/port from the hash table.
+             * It will be ignored due to the HOSTDELETED flag and will
+             * be removed when h_TossStuff_r() cleans up the HOSTDELETED
+             * host.  Removing it here will only result in a search for 
+             * the host/addr/port in the hash chain which will fail.
+             */
+        } else {
+            ViceLog(0,
+                    ("Removing address that does not belong to host %" AFS_PTR_FMT " (%s:%d).\n",
+                     host, afs_inet_ntoa_r(host->host, hoststr), ntohs(host->port)));
         }
     } else {
-        removeInterfaceAddr_r(host, host->host, host->port);
-        if (host->interface->numberOfInterfaces == 0) {
-            ViceLog(25,
-                     ("Removed only address for host %x (%s:%d), no alternate interfaces, deleting host.\n",
-                       host, afs_inet_ntoa_r(host->host, hoststr), ntohs(host->port)));
-            host->hostFlags |= HOSTDELETED;
-        } else {
-            struct rx_connection *rxconn;
+        if (host->host == addr && host->port == port)  {
+            removeInterfaceAddr_r(host, addr, port);
 
             rxconn = host->callback_rxcon;
             host->callback_rxcon = NULL;
@@ -1365,11 +1450,11 @@ removeAddress_r(struct host *host, afs_uint32 addr, afs_uint16 port)
             if (rxconn) {
                 struct client *client;
                 /*
-                * If rx_DestroyConnection calls h_FreeConnection we will
-                * deadlock on the host_glock_mutex. Work around the problem
-                * by unhooking the client from the connection before
-                * destroying the connection.
-                */
+                 * If rx_DestroyConnection calls h_FreeConnection we will
+                 * deadlock on the host_glock_mutex. Work around the problem
+                 * by unhooking the client from the connection before
+                 * destroying the connection.
+                 */
                 client = rx_GetSpecific(rxconn, rxcon_client_key);
                 rx_SetSpecific(rxconn, rxcon_client_key, (void *)0);
                 rx_DestroyConnection(rxconn);
@@ -1378,7 +1463,7 @@ removeAddress_r(struct host *host, afs_uint32 addr, afs_uint16 port)
             for (i=0; i < host->interface->numberOfInterfaces; i++) {
                 if (host->interface->interface[i].valid) {
                     ViceLog(25,
-                             ("Removed address for host %x (%s:%d), new primary interface %s:%d.\n",
+                             ("Removed address for host %" AFS_PTR_FMT " (%s:%d), new primary interface %s:%d.\n",
                                host, afs_inet_ntoa_r(host->host, hoststr), ntohs(host->port),
                                afs_inet_ntoa_r(host->interface->interface[i].addr, hoststr2), 
                                ntohs(host->interface->interface[i].port)));
@@ -1391,9 +1476,12 @@ removeAddress_r(struct host *host, afs_uint32 addr, afs_uint16 port)
 
             if (i == host->interface->numberOfInterfaces) {
                 ViceLog(25,
-                         ("Removed only address for host %x (%s:%d), no valid alternate interfaces, deleting host.\n",
+                         ("Removed only address for host %" AFS_PTR_FMT " (%s:%d), no valid alternate interfaces, deleting host.\n",
                            host, afs_inet_ntoa_r(host->host, hoststr), ntohs(host->port)));
                 host->hostFlags |= HOSTDELETED;
+                /* addr/port was removed from the hash table */
+                host->host = 0;
+                host->port = 0;
             } else {
                 if (!sc)
                     sc = rxnull_NewClientSecurityObject();
@@ -1402,6 +1490,9 @@ removeAddress_r(struct host *host, afs_uint32 addr, afs_uint16 port)
                 rx_SetConnDeadTime(host->callback_rxcon, 50);
                 rx_SetConnHardDeadTime(host->callback_rxcon, AFS_HARDDEADTIME);
             }
+        } else {
+            /* not the primary addr/port, just invalidate it */
+            invalidateInterfaceAddr_r(host, addr, port);
         }
     }
 
@@ -1480,7 +1571,7 @@ h_GetHost_r(struct rx_connection *tcon)
            if (!held)
                h_Release_r(host);
            ViceLog(125,
-                   ("Host %x (%s:%d) starting h_Lookup again\n",
+                   ("Host %" AFS_PTR_FMT " (%s:%d) starting h_Lookup again\n",
                     host, afs_inet_ntoa_r(host->host, hoststr),
                     ntohs(host->port)));
            goto retry;
@@ -1545,7 +1636,7 @@ h_GetHost_r(struct rx_connection *tcon)
                 * that we maintain some extra callback state information */
                if (host->interface) {
                    ViceLog(0,
-                           ("Host %x (%s:%d) used to support WhoAreYou, deleting.\n",
+                           ("Host %" AFS_PTR_FMT " (%s:%d) used to support WhoAreYou, deleting.\n",
                             host, 
                             afs_inet_ntoa_r(host->host, hoststr),
                             ntohs(host->port)));
@@ -1596,7 +1687,7 @@ h_GetHost_r(struct rx_connection *tcon)
                     removeAddress_r(host, haddr, hport);
                } else {
                    ViceLog(25,
-                           ("Uuid doesn't match host %x (%s:%d).\n",
+                           ("Uuid doesn't match host %" AFS_PTR_FMT " (%s:%d).\n",
                             host, afs_inet_ntoa_r(host->host, hoststr), ntohs(host->port)));
                    
                    removeAddress_r(host, host->host, host->port);
@@ -1634,13 +1725,18 @@ h_GetHost_r(struct rx_connection *tcon)
                      * callback connection, and destroy the old one.
                      */
                     struct rx_connection *rxconn;
-                    ViceLog(0,("CB: ProbeUuid for host %x (%s:%d) failed %d\n",
+                    ViceLog(0,("CB: ProbeUuid for host %" AFS_PTR_FMT " (%s:%d) failed %d\n",
                               host, 
                               afs_inet_ntoa_r(host->host, hoststr),
                               ntohs(host->port),code2));
-                   
-                    removeInterfaceAddr_r(host, host->host, host->port);
+
+                    /* 
+                     * make sure we add and then remove.  otherwise, we
+                     * might end up with no valid interfaces after the 
+                     * remove and the host will have been marked deleted.
+                     */
                     addInterfaceAddr_r(host, haddr, hport);
+                    removeInterfaceAddr_r(host, host->host, host->port);
                     host->host = haddr;
                     host->port = hport;
                     rxconn = host->callback_rxcon;
@@ -1681,7 +1777,7 @@ h_GetHost_r(struct rx_connection *tcon)
                 return 0;
            } else {
                ViceLog(0,
-                       ("CB: WhoAreYou failed for host %x (%s:%d), error %d\n",
+                       ("CB: WhoAreYou failed for host %" AFS_PTR_FMT " (%s:%d), error %d\n",
                         host, afs_inet_ntoa_r(host->host, hoststr),
                         ntohs(host->port), code));
                host->hostFlags |= VENUSDOWN;
@@ -1699,7 +1795,7 @@ h_GetHost_r(struct rx_connection *tcon)
        if (!(host->hostFlags & ALTADDR)) {
            /* another thread is doing the initialisation */
            ViceLog(125,
-                   ("Host %x (%s:%d) waiting for host-init to complete\n",
+                   ("Host %" AFS_PTR_FMT " (%s:%d) waiting for host-init to complete\n",
                     host, afs_inet_ntoa_r(host->host, hoststr),
                     ntohs(host->port)));
            h_Lock_r(host);
@@ -1707,7 +1803,7 @@ h_GetHost_r(struct rx_connection *tcon)
            if (!held)
                h_Release_r(host);
            ViceLog(125,
-                   ("Host %x (%s:%d) starting h_Lookup again\n",
+                   ("Host %" AFS_PTR_FMT " (%s:%d) starting h_Lookup again\n",
                     host, afs_inet_ntoa_r(host->host, hoststr),
                     ntohs(host->port)));
            goto retry;
@@ -1727,7 +1823,7 @@ h_GetHost_r(struct rx_connection *tcon)
            if (host->interface)
                afsUUID_to_string(&host->interface->uuid, uuid2, 127);
            ViceLog(0,
-                   ("CB: new identity for host %x (%s:%d), deleting(%x %x %s %s)\n",
+                   ("CB: new identity for host %" AFS_PTR_FMT " (%s:%d), deleting(%x %x %s %s)\n",
                     host, afs_inet_ntoa_r(host->host, hoststr), ntohs(host->port),
                     identP->valid, host->interface,
                     identP->valid ? uuid1 : "no_uuid",
@@ -1773,7 +1869,7 @@ h_GetHost_r(struct rx_connection *tcon)
                if (!pident)
                    rx_SetSpecific(tcon, rxcon_ident_key, identP);
                ViceLog(25,
-                       ("Host %x (%s:%d) does not support WhoAreYou.\n",
+                       ("Host %" AFS_PTR_FMT " (%s:%d) does not support WhoAreYou.\n",
                         host, afs_inet_ntoa_r(host->host, hoststr),
                         ntohs(host->port)));
                code = 0;
@@ -1794,7 +1890,7 @@ h_GetHost_r(struct rx_connection *tcon)
                if (!pident)
                    rx_SetSpecific(tcon, rxcon_ident_key, identP);
                ViceLog(25,
-                       ("WhoAreYou success on host %x (%s:%d)\n",
+                       ("WhoAreYou success on host %" AFS_PTR_FMT " (%s:%d)\n",
                         host, afs_inet_ntoa_r(host->host, hoststr),
                         ntohs(host->port)));
            }
@@ -1836,7 +1932,7 @@ h_GetHost_r(struct rx_connection *tcon)
                             * MultiProbeAlternateAddress_r() will remove the
                             * alternate interfaces that do not have the same
                             * Uuid. */
-                           ViceLog(0,("CB: ProbeUuid for host %x (%s:%d) failed %d\n",
+                           ViceLog(0,("CB: ProbeUuid for host %" AFS_PTR_FMT " (%s:%d) failed %d\n",
                                         oldHost, 
                                          afs_inet_ntoa_r(oldHost->host, hoststr),
                                         ntohs(oldHost->port),code2));
@@ -1858,39 +1954,46 @@ h_GetHost_r(struct rx_connection *tcon)
                        struct rx_connection *rxconn;
 
                        ViceLog(25,
-                                 ("CB: Host %x (%s:%d) has new addr %s:%d\n",
+                                 ("CB: Host %" AFS_PTR_FMT " (%s:%d) has new addr %s:%d\n",
                                    oldHost, 
                                    afs_inet_ntoa_r(oldHost->host, hoststr2),
                                    ntohs(oldHost->port),
                                    afs_inet_ntoa_r(haddr, hoststr),
                                    ntohs(hport)));
+                       /* 
+                        * add then remove.  otherwise the host may get marked
+                        * deleted if we removed the only valid address.
+                        */
+                       addInterfaceAddr_r(oldHost, haddr, hport);
                        if (probefail || oldHost->host == haddr) {
-                           /* The probe failed which means that the old address is 
-                            * either unreachable or is not the same host we were just
-                            * contacted by.  We will also remove addresses if only
-                            * the port has changed because that indicates the client
+                           /* 
+                            * The probe failed which means that the old 
+                            * address is either unreachable or is not the 
+                            * same host we were just contacted by.  We will 
+                            * also remove addresses if only the port has 
+                            * changed because that indicates the client
                             * is behind a NAT. 
                             */
                            removeInterfaceAddr_r(oldHost, oldHost->host, oldHost->port);
                        } else {
-                           int i, found;
+                           int i;
                            struct Interface *interface = oldHost->interface;
                            int number = oldHost->interface->numberOfInterfaces;
-                           for (i = 0, found = 0; i < number; i++) {
+                           for (i = 0; i < number; i++) {
                                if (interface->interface[i].addr == haddr &&
                                    interface->interface[i].port != hport) {
-                                   found = 1;
+                                   /* 
+                                    * We have just been contacted by a client
+                                    * that has been seen from behind a NAT 
+                                    * and at least one other address.
+                                    */
+                                   removeInterfaceAddr_r(oldHost, haddr, 
+                                                         interface->interface[i].port);
                                    break;
                                }
                            }
-                           if (found) {
-                               /* We have just been contacted by a client that has been
-                                * seen from behind a NAT and at least one other address.
-                                */
-                               removeInterfaceAddr_r(oldHost, haddr, interface->interface[i].port);
-                           }
                        }
-                       addInterfaceAddr_r(oldHost, haddr, hport);
+                       h_AddHostToAddrHashTable_r(haddr, hport, oldHost);
                        oldHost->host = haddr;
                        oldHost->port = hport;
                        rxconn = oldHost->callback_rxcon;
@@ -1930,7 +2033,7 @@ h_GetHost_r(struct rx_connection *tcon)
                    H_LOCK;
                    if (code == 0) {
                        ViceLog(25,
-                               ("InitCallBackState3 success on host %x (%s:%d)\n",
+                               ("InitCallBackState3 success on host %" AFS_PTR_FMT " (%s:%d)\n",
                                 host, afs_inet_ntoa_r(host->host, hoststr),
                                 ntohs(host->port)));
                        assert(interfValid == 1);
@@ -1940,12 +2043,12 @@ h_GetHost_r(struct rx_connection *tcon)
            }
            if (code) {
                ViceLog(0,
-                       ("CB: RCallBackConnectBack failed for %x (%s:%d)\n",
+                       ("CB: RCallBackConnectBack failed for %" AFS_PTR_FMT " (%s:%d)\n",
                         host, afs_inet_ntoa_r(host->host, hoststr), ntohs(host->port)));
                host->hostFlags |= VENUSDOWN;
            } else {
                ViceLog(125,
-                       ("CB: RCallBackConnectBack succeeded for %x (%s:%d)\n",
+                       ("CB: RCallBackConnectBack succeeded for %" AFS_PTR_FMT " (%s:%d)\n",
                         host, afs_inet_ntoa_r(host->host, hoststr), ntohs(host->port)));
                host->hostFlags |= RESETDONE;
            }
@@ -2272,7 +2375,7 @@ h_FindClient_r(struct rx_connection *tcon)
            if (code) {
                char hoststr[16];
                ViceLog(0,
-                       ("pr_GetCPS failed(%d) for user %d, host %x (%s:%d)\n",
+                       ("pr_GetCPS failed(%d) for user %d, host %" AFS_PTR_FMT " (%s:%d)\n",
                         code, viceid, client->host, 
                          afs_inet_ntoa_r(client->host->host,hoststr),
                         ntohs(client->host->port)));
@@ -3076,13 +3179,20 @@ h_stateRestoreHost(struct fs_dump_state * state)
     h_diskEntryToHost_r(&hdsk, host);
     h_SetupCallbackConn_r(host);
 
+    h_AddHostToAddrHashTable_r(host->host, host->port, host);
     if (ifp) {
        int i;
        for (i = ifp->numberOfInterfaces-1; i >= 0; i--) {
+            if (ifp->interface[i].valid && 
+                !(ifp->interface[i].addr == host->host &&
+                  ifp->interface[i].port == host->port)) {
+                h_AddHostToAddrHashTable_r(ifp->interface[i].addr, 
+                                           ifp->interface[i].port, 
+                                           host);
+            }
        }
        h_AddHostToUuidHashTable_r(&ifp->uuid, host);
     }
-    h_AddHostToAddrHashTable_r(host->host, host->port, host);
     h_InsertList_r(host);
 
     /* setup host id map entry */
@@ -3437,7 +3547,7 @@ CheckHost(register struct host *host, int held, void *rock)
                     * back state, because break delayed callbacks (called when a
                     * message is received from the workstation) will always send a 
                     * break all call backs to the workstation if there is no
-                    *callback.
+                    * callback.
                     */
                }
            } else {
@@ -3540,7 +3650,7 @@ CheckHost_r(register struct host *host, int held, char *dummy)
                     * back state, because break delayed callbacks (called when a
                     * message is received from the workstation) will always send a 
                     * break all call backs to the workstation if there is no
-                    *callback.
+                    * callback.
                     */
                }
            } else {
@@ -3613,9 +3723,8 @@ h_CheckHosts(void)
 
 /*
  * This is called with host locked and held. At this point, the
- * hostAddrHashTable should not have any entries for the alternate
- * interfaces. This function has to insert these entries in the
- * hostAddrHashTable.
+ * hostAddrHashTable has an entry for the primary addr/port inserted
+ * by h_Alloc_r().  No other interfaces should be considered valid.
  *
  * The addresses in the interfaceAddr list are in host byte order.
  */
@@ -3733,7 +3842,8 @@ initInterfaceAddr_r(struct host *host, struct interfaceAddr *interf)
         * are coming from fully connected hosts (no NAT/PATs)
         */
        interface->interface[i].port = port7001;
-        interface->interface[i].valid = 1;      /* valid until a conflict is found */
+        interface->interface[i].valid = 
+            (interf->addr_in[i] == myAddr && port7001 == myPort) ? 1 : 0;
     }
 
     interface->uuid = interf->uuid;
@@ -3741,13 +3851,15 @@ initInterfaceAddr_r(struct host *host, struct interfaceAddr *interf)
     assert(!host->interface);
     host->interface = interface;
 
-    afsUUID_to_string(&interface->uuid, uuidstr, 127);
-
-    ViceLog(125, ("--- uuid %s\n", uuidstr));
-    for (i = 0; i < host->interface->numberOfInterfaces; i++) {
-       ViceLog(125, ("--- alt address %s:%d\n", 
-                      afs_inet_ntoa_r(host->interface->interface[i].addr, hoststr),
-                      ntohs(host->interface->interface[i].port)));
+    if (LogLevel >= 125) {
+       afsUUID_to_string(&interface->uuid, uuidstr, 127);
+       
+       ViceLog(125, ("--- uuid %s\n", uuidstr));
+       for (i = 0; i < host->interface->numberOfInterfaces; i++) {
+           ViceLog(125, ("--- alt address %s:%d\n", 
+                         afs_inet_ntoa_r(host->interface->interface[i].addr, hoststr),
+                         ntohs(host->interface->interface[i].port)));
+       }
     }
 
     return 0;
@@ -3756,23 +3868,32 @@ initInterfaceAddr_r(struct host *host, struct interfaceAddr *interf)
 /* deleted a HashChain structure for this address and host */
 /* returns 1 on success */
 int
-h_DeleteHostFromAddrHashTable_r(afs_uint32 addr, afs_uint16 port, struct host *host)
+h_DeleteHostFromAddrHashTable_r(afs_uint32 addr, afs_uint16 port, 
+                               struct host *host)
 {
-    int flag = 0;
+    char hoststr[16];
     register struct h_AddrHashChain **hp, *th;
 
-    for (hp = &hostAddrHashTable[h_HashIndex(addr)]; (th = *hp);) {
-       assert(th->hostPtr);
-       if (th->hostPtr == host && th->addr == addr && th->port == port) {
-           *hp = th->next;
-           free(th);
-           flag = 1;
-           break;
-       } else {
-           hp = &th->next;
-       }
+    if (addr == 0 && port == 0)
+       return 1;
+
+    for (hp = &hostAddrHashTable[h_HashIndex(addr)]; (th = *hp); 
+        hp = &th->next) {
+        assert(th->hostPtr);
+        if (th->hostPtr == host && th->addr == addr && th->port == port) {
+           ViceLog(125, ("h_DeleteHostFromAddrHashTable_r: host %" AFS_PTR_FMT " (%s:%d)\n",
+                         host, afs_inet_ntoa_r(host->host, hoststr),
+                         ntohs(host->port)));
+            *hp = th->next;
+            free(th);
+           return 1;
+        }
     }
-    return flag;
+    ViceLog(125, 
+           ("h_DeleteHostFromAddrHashTable_r: host %" AFS_PTR_FMT " (%s:%d) not found\n",
+            host, afs_inet_ntoa_r(host->host, hoststr), 
+            ntohs(host->port)));
+    return 0;
 }