rx: Don't malloc the xmit list
[openafs.git] / src / rx / rx.c
index 81daa78..9696e3d 100644 (file)
@@ -70,6 +70,8 @@
 #include "rx_globals.h"
 #include "rx_trace.h"
 #include "rx_atomic.h"
+#include "rx_internal.h"
+#include "rx_stats.h"
 #define        AFSOP_STOP_RXCALLBACK   210     /* Stop CALLBACK process */
 #define        AFSOP_STOP_AFS          211     /* Stop AFS process */
 #define        AFSOP_STOP_BKG          212     /* Stop BKG process */
@@ -108,6 +110,8 @@ extern afs_int32 afs_termState;
 # include "rx_atomic.h"
 # include "rx_globals.h"
 # include "rx_trace.h"
+# include "rx_internal.h"
+# include "rx_stats.h"
 # include <afs/rxgen_consts.h>
 #endif /* KERNEL */
 
@@ -125,14 +129,17 @@ int (*swapNameProgram) (PROCESS, const char *, char *) = 0;
 
 /* Local static routines */
 static void rxi_DestroyConnectionNoLock(struct rx_connection *conn);
+static void rxi_ComputeRoundTripTime(struct rx_packet *, struct clock *,
+                                    struct rx_peer *, struct clock *);
+
 #ifdef RX_ENABLE_LOCKS
 static void rxi_SetAcksInTransmitQueue(struct rx_call *call);
 #endif
 
 #ifdef AFS_GLOBAL_RXLOCK_KERNEL
 struct rx_tq_debug {
-    afs_int32 rxi_start_aborted;       /* rxi_start awoke after rxi_Send in error. */
-    afs_int32 rxi_start_in_error;
+    rx_atomic_t rxi_start_aborted; /* rxi_start awoke after rxi_Send in error.*/
+    rx_atomic_t rxi_start_in_error;
 } rx_tq_debug;
 #endif /* AFS_GLOBAL_RXLOCK_KERNEL */
 
@@ -154,6 +161,9 @@ static unsigned int rxi_rpc_peer_stat_cnt;
 
 static unsigned int rxi_rpc_process_stat_cnt;
 
+rx_atomic_t rx_nWaiting = RX_ATOMIC_INIT(0);
+rx_atomic_t rx_nWaited = RX_ATOMIC_INIT(0);
+
 #if !defined(offsetof)
 #include <stddef.h>            /* for definition of offsetof() */
 #endif
@@ -170,11 +180,10 @@ afs_kmutex_t rx_atomic_mutex;
  * to ease NT porting
  */
 
-extern afs_kmutex_t rx_stats_mutex;
-extern afs_kmutex_t rx_waiting_mutex;
 extern afs_kmutex_t rx_quota_mutex;
 extern afs_kmutex_t rx_pthread_mutex;
 extern afs_kmutex_t rx_packets_mutex;
+extern afs_kmutex_t rx_refcnt_mutex;
 extern afs_kmutex_t des_init_mutex;
 extern afs_kmutex_t des_random_mutex;
 extern afs_kmutex_t rx_clock_mutex;
@@ -201,11 +210,11 @@ rxi_InitPthread(void)
 {
     MUTEX_INIT(&rx_clock_mutex, "clock", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_stats_mutex, "stats", MUTEX_DEFAULT, 0);
-    MUTEX_INIT(&rx_waiting_mutex, "waiting", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_atomic_mutex, "atomic", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_quota_mutex, "quota", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_pthread_mutex, "pthread", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_packets_mutex, "packets", MUTEX_DEFAULT, 0);
+    MUTEX_INIT(&rx_refcnt_mutex, "refcnts", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&epoch_mutex, "epoch", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_init_mutex, "init", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_event_mutex, "event", MUTEX_DEFAULT, 0);
@@ -352,7 +361,9 @@ struct rx_connection *rxLastConn = 0;
  * lowest level:
  *     multi_handle->lock
  *     rxevent_lock
+ *      rx_packets_mutex
  *     rx_stats_mutex
+ *      rx_refcnt_mutex
  *     rx_atomic_mutex
  *
  * Do we need a lock to protect the peer field in the conn structure?
@@ -478,10 +489,10 @@ rx_InitHost(u_int host, u_int port)
     rxdb_init();
 #endif /* RX_LOCKS_DB */
     MUTEX_INIT(&rx_stats_mutex, "rx_stats_mutex", MUTEX_DEFAULT, 0);
-    MUTEX_INIT(&rx_waiting_mutex, "rx_waiting_mutex", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_quota_mutex, "rx_quota_mutex", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_pthread_mutex, "rx_pthread_mutex", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_packets_mutex, "rx_packets_mutex", MUTEX_DEFAULT, 0);
+    MUTEX_INIT(&rx_refcnt_mutex, "rx_refcnt_mutex", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_rpc_stats, "rx_rpc_stats", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_freePktQ_lock, "rx_freePktQ_lock", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&freeSQEList_lock, "freeSQEList lock", MUTEX_DEFAULT, 0);
@@ -503,7 +514,7 @@ rx_InitHost(u_int host, u_int port)
     rxi_nCalls = 0;
     rx_connDeadTime = 12;
     rx_tranquil = 0;           /* reset flag */
-    memset(&rx_stats, 0, sizeof(struct rx_statistics));
+    rxi_ResetStatistics();
     htable = (char *)
        osi_Alloc(rx_hashTableSize * sizeof(struct rx_connection *));
     PIN(htable, rx_hashTableSize * sizeof(struct rx_connection *));    /* XXXXX */
@@ -865,21 +876,69 @@ rx_NewConnection(afs_uint32 shost, u_short sport, u_short sservice,
     conn->next = rx_connHashTable[hashindex];
     rx_connHashTable[hashindex] = conn;
     if (rx_stats_active)
-        rx_MutexIncrement(rx_stats.nClientConns, rx_stats_mutex);
+       rx_atomic_inc(&rx_stats.nClientConns);
     MUTEX_EXIT(&rx_connHashTable_lock);
     USERPRI;
     return conn;
 }
 
+/**
+ * Ensure a connection's timeout values are valid.
+ *
+ * @param[in] conn The connection to check
+ *
+ * @post conn->secondUntilDead <= conn->idleDeadTime <= conn->hardDeadTime,
+ *       unless idleDeadTime and/or hardDeadTime are not set
+ * @internal
+ */
+static void
+rxi_CheckConnTimeouts(struct rx_connection *conn)
+{
+    /* a connection's timeouts must have the relationship
+     * deadTime <= idleDeadTime <= hardDeadTime. Otherwise, for example, a
+     * total loss of network to a peer may cause an idle timeout instead of a
+     * dead timeout, simply because the idle timeout gets hit first. Also set
+     * a minimum deadTime of 6, just to ensure it doesn't get set too low. */
+    /* this logic is slightly complicated by the fact that
+     * idleDeadTime/hardDeadTime may not be set at all, but it's not too bad.
+     */
+    conn->secondsUntilDead = MAX(conn->secondsUntilDead, 6);
+    if (conn->idleDeadTime) {
+       conn->idleDeadTime = MAX(conn->idleDeadTime, conn->secondsUntilDead);
+    }
+    if (conn->hardDeadTime) {
+       if (conn->idleDeadTime) {
+           conn->hardDeadTime = MAX(conn->idleDeadTime, conn->hardDeadTime);
+       } else {
+           conn->hardDeadTime = MAX(conn->secondsUntilDead, conn->hardDeadTime);
+       }
+    }
+}
+
 void
 rx_SetConnDeadTime(struct rx_connection *conn, int seconds)
 {
     /* The idea is to set the dead time to a value that allows several
      * keepalives to be dropped without timing out the connection. */
-    conn->secondsUntilDead = MAX(seconds, 6);
+    conn->secondsUntilDead = seconds;
+    rxi_CheckConnTimeouts(conn);
     conn->secondsUntilPing = conn->secondsUntilDead / 6;
 }
 
+void
+rx_SetConnHardDeadTime(struct rx_connection *conn, int seconds)
+{
+    conn->hardDeadTime = seconds;
+    rxi_CheckConnTimeouts(conn);
+}
+
+void
+rx_SetConnIdleDeadTime(struct rx_connection *conn, int seconds)
+{
+    conn->idleDeadTime = seconds;
+    rxi_CheckConnTimeouts(conn);
+}
+
 int rxi_lowPeerRefCount = 0;
 int rxi_lowConnRefCount = 0;
 
@@ -920,9 +979,9 @@ rxi_CleanupConnection(struct rx_connection *conn)
     if (rx_stats_active)
     {
         if (conn->type == RX_SERVER_CONNECTION)
-            rx_MutexDecrement(rx_stats.nServerConns, rx_stats_mutex);
+           rx_atomic_dec(&rx_stats.nServerConns);
         else
-            rx_MutexDecrement(rx_stats.nClientConns, rx_stats_mutex);
+           rx_atomic_dec(&rx_stats.nClientConns);
     }
 #ifndef KERNEL
     if (conn->specific) {
@@ -977,6 +1036,7 @@ rxi_DestroyConnectionNoLock(struct rx_connection *conn)
 
     NETPRI;
     MUTEX_ENTER(&conn->conn_data_lock);
+    MUTEX_ENTER(&rx_refcnt_mutex);
     if (conn->refCount > 0)
        conn->refCount--;
     else {
@@ -989,6 +1049,7 @@ rxi_DestroyConnectionNoLock(struct rx_connection *conn)
 
     if ((conn->refCount > 0) || (conn->flags & RX_CONN_BUSY)) {
        /* Busy; wait till the last guy before proceeding */
+        MUTEX_EXIT(&rx_refcnt_mutex);
        MUTEX_EXIT(&conn->conn_data_lock);
        USERPRI;
        return;
@@ -1004,6 +1065,7 @@ rxi_DestroyConnectionNoLock(struct rx_connection *conn)
        USERPRI;
        return;
     }
+    MUTEX_EXIT(&rx_refcnt_mutex);
     MUTEX_EXIT(&conn->conn_data_lock);
 
     /* Check for extant references to this connection */
@@ -1116,9 +1178,9 @@ rx_GetConnection(struct rx_connection *conn)
     SPLVAR;
 
     NETPRI;
-    MUTEX_ENTER(&conn->conn_data_lock);
+    MUTEX_ENTER(&rx_refcnt_mutex);
     conn->refCount++;
-    MUTEX_EXIT(&conn->conn_data_lock);
+    MUTEX_EXIT(&rx_refcnt_mutex);
     USERPRI;
 }
 
@@ -1224,8 +1286,10 @@ rx_NewCall(struct rx_connection *conn)
                          * effect on overall system performance.
                          */
                         call->state = RX_STATE_RESET;
-                        CALL_HOLD(call, RX_CALL_REFCOUNT_BEGIN);
                         MUTEX_EXIT(&conn->conn_call_lock);
+                        MUTEX_ENTER(&rx_refcnt_mutex);
+                        CALL_HOLD(call, RX_CALL_REFCOUNT_BEGIN);
+                        MUTEX_EXIT(&rx_refcnt_mutex);
                         rxi_ResetCall(call, 0);
                         (*call->callNumber)++;
                         if (MUTEX_TRYENTER(&conn->conn_call_lock))
@@ -1257,7 +1321,9 @@ rx_NewCall(struct rx_connection *conn)
                          * Instead, cycle through one more time to see if
                          * we can find a call that can call our own.
                          */
+                        MUTEX_ENTER(&rx_refcnt_mutex);
                         CALL_RELE(call, RX_CALL_REFCOUNT_BEGIN);
+                        MUTEX_EXIT(&rx_refcnt_mutex);
                         wait = 0;
                     }
                     MUTEX_EXIT(&call->lock);
@@ -1265,7 +1331,9 @@ rx_NewCall(struct rx_connection *conn)
            } else {
                 /* rxi_NewCall returns with mutex locked */
                call = rxi_NewCall(conn, i);
+                MUTEX_ENTER(&rx_refcnt_mutex);
                 CALL_HOLD(call, RX_CALL_REFCOUNT_BEGIN);
+                MUTEX_EXIT(&rx_refcnt_mutex);
                break;
            }
        }
@@ -1755,9 +1823,7 @@ rx_GetCall(int tno, struct rx_service *cur_service, osi_socket * socketp)
 
            if (call->flags & RX_CALL_WAIT_PROC) {
                call->flags &= ~RX_CALL_WAIT_PROC;
-                MUTEX_ENTER(&rx_waiting_mutex);
-                rx_nWaiting--;
-                MUTEX_EXIT(&rx_waiting_mutex);
+               rx_atomic_dec(&rx_nWaiting);
            }
 
            if (call->state != RX_STATE_PRECALL || call->error) {
@@ -1834,8 +1900,10 @@ rx_GetCall(int tno, struct rx_service *cur_service, osi_socket * socketp)
             call->conn->service->servicePort, call->conn->service->serviceId,
             call));
 
-       CALL_HOLD(call, RX_CALL_REFCOUNT_BEGIN);
        MUTEX_EXIT(&call->lock);
+        MUTEX_ENTER(&rx_refcnt_mutex);
+       CALL_HOLD(call, RX_CALL_REFCOUNT_BEGIN);
+        MUTEX_EXIT(&rx_refcnt_mutex);
     } else {
        dpf(("rx_GetCall(socketp=%p, *socketp=0x%x)\n", socketp, *socketp));
     }
@@ -1939,7 +2007,7 @@ rx_GetCall(int tno, struct rx_service *cur_service, osi_socket * socketp)
            rxi_minDeficit--;
        rxi_availProcs--;
         MUTEX_EXIT(&rx_quota_mutex);
-       rx_nWaiting--;
+       rx_atomic_dec(&rx_nWaiting);
        /* MUTEX_EXIT(&call->lock); */
     } else {
        /* If there are no eligible incoming calls, add this process
@@ -2049,6 +2117,7 @@ rx_EndCall(struct rx_call *call, afs_int32 rc)
     call->arrivalProc = (void (*)())0;
     if (rc && call->error == 0) {
        rxi_CallError(call, rc);
+        call->mode = RX_MODE_ERROR;
        /* Send an abort message to the peer if this error code has
         * only just been set.  If it was set previously, assume the
         * peer has already been sent the error code or will request it
@@ -2058,10 +2127,14 @@ rx_EndCall(struct rx_call *call, afs_int32 rc)
     if (conn->type == RX_SERVER_CONNECTION) {
        /* Make sure reply or at least dummy reply is sent */
        if (call->mode == RX_MODE_RECEIVING) {
+           MUTEX_EXIT(&call->lock);
            rxi_WriteProc(call, 0, 0);
+           MUTEX_ENTER(&call->lock);
        }
        if (call->mode == RX_MODE_SENDING) {
+            MUTEX_EXIT(&call->lock);
            rxi_FlushWrite(call);
+            MUTEX_ENTER(&call->lock);
        }
        rxi_calltrace(RX_CALL_END, call);
        /* Call goes to hold state until reply packets are acknowledged */
@@ -2080,7 +2153,9 @@ rx_EndCall(struct rx_call *call, afs_int32 rc)
         * no reply arguments are expected */
        if ((call->mode == RX_MODE_SENDING)
            || (call->mode == RX_MODE_RECEIVING && call->rnext == 1)) {
+           MUTEX_EXIT(&call->lock);
            (void)rxi_ReadProc(call, &dummy, 1);
+           MUTEX_ENTER(&call->lock);
        }
 
        /* If we had an outstanding delayed ack, be nice to the server
@@ -2143,9 +2218,11 @@ rx_EndCall(struct rx_call *call, afs_int32 rc)
     call->iovqc -=
 #endif /* RXDEBUG_PACKET */
         rxi_FreePackets(0, &call->iovq);
+    MUTEX_EXIT(&call->lock);
 
+    MUTEX_ENTER(&rx_refcnt_mutex);
     CALL_RELE(call, RX_CALL_REFCOUNT_BEGIN);
-    MUTEX_EXIT(&call->lock);
+    MUTEX_EXIT(&rx_refcnt_mutex);
     if (conn->type == RX_CLIENT_CONNECTION) {
        MUTEX_ENTER(&conn->conn_data_lock);
        conn->flags &= ~RX_CONN_BUSY;
@@ -2189,9 +2266,9 @@ rx_Finalize(void)
            for (conn = *conn_ptr; conn; conn = next) {
                next = conn->next;
                if (conn->type == RX_CLIENT_CONNECTION) {
-                   /* MUTEX_ENTER(&conn->conn_data_lock); when used in kernel */
+                    MUTEX_ENTER(&rx_refcnt_mutex);
                    conn->refCount++;
-                   /* MUTEX_EXIT(&conn->conn_data_lock); when used in kernel */
+                    MUTEX_EXIT(&rx_refcnt_mutex);
 #ifdef RX_ENABLE_LOCKS
                    rxi_DestroyConnectionNoLock(conn);
 #else /* RX_ENABLE_LOCKS */
@@ -2307,7 +2384,7 @@ rxi_NewCall(struct rx_connection *conn, int channel)
 #endif /* AFS_GLOBAL_RXLOCK_KERNEL */
        queue_Remove(call);
         if (rx_stats_active)
-            rx_MutexDecrement(rx_stats.nFreeCallStructs, rx_stats_mutex);
+           rx_atomic_dec(&rx_stats.nFreeCallStructs);
        MUTEX_EXIT(&rx_freeCallQueue_lock);
        MUTEX_ENTER(&call->lock);
        CLEAR_CALL_QUEUE_LOCK(call);
@@ -2329,8 +2406,10 @@ rxi_NewCall(struct rx_connection *conn, int channel)
         call->allNextp = rx_allCallsp;
         rx_allCallsp = call;
         call->call_id =
+           rx_atomic_inc_and_read(&rx_stats.nCallStructs);
+#else /* RXDEBUG_PACKET */
+        rx_atomic_inc(&rx_stats.nCallStructs);
 #endif /* RXDEBUG_PACKET */
-            rx_MutexIncrement(rx_stats.nCallStructs, rx_stats_mutex);
 
         MUTEX_EXIT(&rx_freeCallQueue_lock);
        MUTEX_INIT(&call->lock, "call lock", MUTEX_DEFAULT, NULL);
@@ -2369,16 +2448,12 @@ rxi_NewCall(struct rx_connection *conn, int channel)
 /* A call has been inactive long enough that so we can throw away
  * state, including the call structure, which is placed on the call
  * free list.
- * Call is locked upon entry.
- * haveCTLock set if called from rxi_ReapConnections
+ *
+ * call->lock amd rx_refcnt_mutex are held upon entry.
+ * haveCTLock is set when called from rxi_ReapConnections.
  */
-#ifdef RX_ENABLE_LOCKS
 void
 rxi_FreeCall(struct rx_call *call, int haveCTLock)
-#else /* RX_ENABLE_LOCKS */
-void
-rxi_FreeCall(struct rx_call *call)
-#endif                         /* RX_ENABLE_LOCKS */
 {
     int channel = call->channel;
     struct rx_connection *conn = call->conn;
@@ -2388,6 +2463,7 @@ rxi_FreeCall(struct rx_call *call)
        (*call->callNumber)++;
     rxi_ResetCall(call, 0);
     call->conn->call[channel] = (struct rx_call *)0;
+    MUTEX_EXIT(&rx_refcnt_mutex);
 
     MUTEX_ENTER(&rx_freeCallQueue_lock);
     SET_CALL_QUEUE_LOCK(call, &rx_freeCallQueue_lock);
@@ -2404,7 +2480,7 @@ rxi_FreeCall(struct rx_call *call)
     queue_Append(&rx_freeCallQueue, call);
 #endif /* AFS_GLOBAL_RXLOCK_KERNEL */
     if (rx_stats_active)
-        rx_MutexIncrement(rx_stats.nFreeCallStructs, rx_stats_mutex);
+       rx_atomic_inc(&rx_stats.nFreeCallStructs);
     MUTEX_EXIT(&rx_freeCallQueue_lock);
 
     /* Destroy the connection if it was previously slated for
@@ -2421,7 +2497,9 @@ rxi_FreeCall(struct rx_call *call)
      */
     MUTEX_ENTER(&conn->conn_data_lock);
     if (conn->flags & RX_CONN_DESTROY_ME && !(conn->flags & RX_CONN_MAKECALL_WAITING)) {
+        MUTEX_ENTER(&rx_refcnt_mutex);
        conn->refCount++;
+        MUTEX_EXIT(&rx_refcnt_mutex);
        MUTEX_EXIT(&conn->conn_data_lock);
 #ifdef RX_ENABLE_LOCKS
        if (haveCTLock)
@@ -2434,16 +2512,21 @@ rxi_FreeCall(struct rx_call *call)
     } else {
        MUTEX_EXIT(&conn->conn_data_lock);
     }
+    MUTEX_ENTER(&rx_refcnt_mutex);
 }
 
-afs_int32 rxi_Alloccnt = 0, rxi_Allocsize = 0;
+rx_atomic_t rxi_Allocsize = RX_ATOMIC_INIT(0);
+rx_atomic_t rxi_Alloccnt = RX_ATOMIC_INIT(0);
+
 void *
 rxi_Alloc(size_t size)
 {
     char *p;
 
-    if (rx_stats_active)
-        rx_MutexAdd1Increment2(rxi_Allocsize, (afs_int32)size, rxi_Alloccnt, rx_stats_mutex);
+    if (rx_stats_active) {
+       rx_atomic_add(&rxi_Allocsize, (int) size);
+       rx_atomic_inc(&rxi_Alloccnt);
+    }
 
 p = (char *)
 #if defined(KERNEL) && !defined(UKERNEL) && defined(AFS_FBSD80_ENV)
@@ -2460,8 +2543,10 @@ p = (char *)
 void
 rxi_Free(void *addr, size_t size)
 {
-    if (rx_stats_active)
-        rx_MutexAdd1Decrement2(rxi_Allocsize, -(afs_int32)size, rxi_Alloccnt, rx_stats_mutex);
+    if (rx_stats_active) {
+       rx_atomic_sub(&rxi_Allocsize, (int) size);
+        rx_atomic_dec(&rxi_Alloccnt);
+    }
     osi_Free(addr, size);
 }
 
@@ -2559,7 +2644,7 @@ rxi_FindPeer(afs_uint32 host, u_short port,
            rx_peerHashTable[hashIndex] = pp;
            rxi_InitPeerParams(pp);
             if (rx_stats_active)
-                rx_MutexIncrement(rx_stats.nPeerStructs, rx_stats_mutex);
+               rx_atomic_inc(&rx_stats.nPeerStructs);
        }
     }
     if (pp && create) {
@@ -2670,12 +2755,12 @@ rxi_FindConnection(osi_socket socket, afs_uint32 host,
        if (service->newConnProc)
            (*service->newConnProc) (conn);
         if (rx_stats_active)
-            rx_MutexIncrement(rx_stats.nServerConns, rx_stats_mutex);
+            rx_atomic_inc(&rx_stats.nServerConns);
     }
 
-    MUTEX_ENTER(&conn->conn_data_lock);
+    MUTEX_ENTER(&rx_refcnt_mutex);
     conn->refCount++;
-    MUTEX_EXIT(&conn->conn_data_lock);
+    MUTEX_EXIT(&rx_refcnt_mutex);
 
     rxLastConn = conn;         /* store this connection as the last conn used */
     MUTEX_EXIT(&rx_connHashTable_lock);
@@ -2786,7 +2871,9 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
        MUTEX_ENTER(&conn->conn_data_lock);
        if (np->header.type != RX_PACKET_TYPE_ABORT)
            np = rxi_SendConnectionAbort(conn, np, 1, 0);
+        MUTEX_ENTER(&rx_refcnt_mutex);
        conn->refCount--;
+        MUTEX_EXIT(&rx_refcnt_mutex);
        MUTEX_EXIT(&conn->conn_data_lock);
        return np;
     }
@@ -2799,30 +2886,30 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
            afs_int32 errcode = ntohl(rx_GetInt32(np, 0));
            dpf(("rxi_ReceivePacket ABORT rx_GetInt32 = %d", errcode));
            rxi_ConnectionError(conn, errcode);
-           MUTEX_ENTER(&conn->conn_data_lock);
+            MUTEX_ENTER(&rx_refcnt_mutex);
            conn->refCount--;
-           MUTEX_EXIT(&conn->conn_data_lock);
+            MUTEX_EXIT(&rx_refcnt_mutex);
            return np;
        }
        case RX_PACKET_TYPE_CHALLENGE:
            tnp = rxi_ReceiveChallengePacket(conn, np, 1);
-           MUTEX_ENTER(&conn->conn_data_lock);
+            MUTEX_ENTER(&rx_refcnt_mutex);
            conn->refCount--;
-           MUTEX_EXIT(&conn->conn_data_lock);
+            MUTEX_EXIT(&rx_refcnt_mutex);
            return tnp;
        case RX_PACKET_TYPE_RESPONSE:
            tnp = rxi_ReceiveResponsePacket(conn, np, 1);
-           MUTEX_ENTER(&conn->conn_data_lock);
+            MUTEX_ENTER(&rx_refcnt_mutex);
            conn->refCount--;
-           MUTEX_EXIT(&conn->conn_data_lock);
+            MUTEX_EXIT(&rx_refcnt_mutex);
            return tnp;
        case RX_PACKET_TYPE_PARAMS:
        case RX_PACKET_TYPE_PARAMS + 1:
        case RX_PACKET_TYPE_PARAMS + 2:
            /* ignore these packet types for now */
-           MUTEX_ENTER(&conn->conn_data_lock);
+            MUTEX_ENTER(&rx_refcnt_mutex);
            conn->refCount--;
-           MUTEX_EXIT(&conn->conn_data_lock);
+            MUTEX_EXIT(&rx_refcnt_mutex);
            return np;
 
 
@@ -2832,7 +2919,9 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
            rxi_ConnectionError(conn, RX_PROTOCOL_ERROR);
            MUTEX_ENTER(&conn->conn_data_lock);
            tnp = rxi_SendConnectionAbort(conn, np, 1, 0);
+            MUTEX_ENTER(&rx_refcnt_mutex);
            conn->refCount--;
+            MUTEX_EXIT(&rx_refcnt_mutex);
            MUTEX_EXIT(&conn->conn_data_lock);
            return tnp;
        }
@@ -2869,10 +2958,10 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
             * it must be for the previous call.
             */
             if (rx_stats_active)
-                rx_MutexIncrement(rx_stats.spuriousPacketsRead, rx_stats_mutex);
-           MUTEX_ENTER(&conn->conn_data_lock);
+               rx_atomic_inc(&rx_stats.spuriousPacketsRead);
+            MUTEX_ENTER(&rx_refcnt_mutex);
            conn->refCount--;
-           MUTEX_EXIT(&conn->conn_data_lock);
+            MUTEX_EXIT(&rx_refcnt_mutex);
            return np;
        }
     }
@@ -2882,14 +2971,14 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
     if (type == RX_SERVER_CONNECTION) {        /* We're the server */
        if (np->header.callNumber < currentCallNumber) {
             if (rx_stats_active)
-                rx_MutexIncrement(rx_stats.spuriousPacketsRead, rx_stats_mutex);
+               rx_atomic_inc(&rx_stats.spuriousPacketsRead);
 #ifdef RX_ENABLE_LOCKS
            if (call)
                MUTEX_EXIT(&call->lock);
 #endif
-           MUTEX_ENTER(&conn->conn_data_lock);
+            MUTEX_ENTER(&rx_refcnt_mutex);
            conn->refCount--;
-           MUTEX_EXIT(&conn->conn_data_lock);
+            MUTEX_EXIT(&rx_refcnt_mutex);
            return np;
        }
        if (!call) {
@@ -2912,17 +3001,18 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
             * If the number of queued calls exceeds the overload
             * threshold then abort this call.
             */
-           if ((rx_BusyThreshold > 0) && (rx_nWaiting > rx_BusyThreshold)) {
+           if ((rx_BusyThreshold > 0) &&
+               (rx_atomic_read(&rx_nWaiting) > rx_BusyThreshold)) {
                struct rx_packet *tp;
 
                rxi_CallError(call, rx_BusyError);
                tp = rxi_SendCallAbort(call, np, 1, 0);
                MUTEX_EXIT(&call->lock);
-               MUTEX_ENTER(&conn->conn_data_lock);
+                MUTEX_ENTER(&rx_refcnt_mutex);
                conn->refCount--;
-               MUTEX_EXIT(&conn->conn_data_lock);
+                MUTEX_EXIT(&rx_refcnt_mutex);
                 if (rx_stats_active)
-                    rx_MutexIncrement(rx_stats.nBusies, rx_stats_mutex);
+                    rx_atomic_inc(&rx_stats.nBusies);
                return tp;
            }
            rxi_KeepAliveOn(call);
@@ -2958,9 +3048,9 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
                tp = rxi_SendSpecial(call, conn, np, RX_PACKET_TYPE_BUSY,
                                     NULL, 0, 1);
                MUTEX_EXIT(&call->lock);
-               MUTEX_ENTER(&conn->conn_data_lock);
+                MUTEX_ENTER(&rx_refcnt_mutex);
                conn->refCount--;
-               MUTEX_EXIT(&conn->conn_data_lock);
+                MUTEX_EXIT(&rx_refcnt_mutex);
                return tp;
            }
            rxi_ResetCall(call, 0);
@@ -2980,17 +3070,18 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
             * If the number of queued calls exceeds the overload
             * threshold then abort this call.
             */
-           if ((rx_BusyThreshold > 0) && (rx_nWaiting > rx_BusyThreshold)) {
+           if ((rx_BusyThreshold > 0) &&
+               (rx_atomic_read(&rx_nWaiting) > rx_BusyThreshold)) {
                struct rx_packet *tp;
 
                rxi_CallError(call, rx_BusyError);
                tp = rxi_SendCallAbort(call, np, 1, 0);
                MUTEX_EXIT(&call->lock);
-               MUTEX_ENTER(&conn->conn_data_lock);
+                MUTEX_ENTER(&rx_refcnt_mutex);
                conn->refCount--;
-               MUTEX_EXIT(&conn->conn_data_lock);
+                MUTEX_EXIT(&rx_refcnt_mutex);
                 if (rx_stats_active)
-                    rx_MutexIncrement(rx_stats.nBusies, rx_stats_mutex);
+                    rx_atomic_inc(&rx_stats.nBusies);
                return tp;
            }
            rxi_KeepAliveOn(call);
@@ -3002,15 +3093,15 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
        if (call && (call->state == RX_STATE_DALLY)
            && (np->header.type == RX_PACKET_TYPE_ACK)) {
             if (rx_stats_active)
-                rx_MutexIncrement(rx_stats.ignorePacketDally, rx_stats_mutex);
+                rx_atomic_inc(&rx_stats.ignorePacketDally);
 #ifdef  RX_ENABLE_LOCKS
            if (call) {
                MUTEX_EXIT(&call->lock);
            }
 #endif
-           MUTEX_ENTER(&conn->conn_data_lock);
+            MUTEX_ENTER(&rx_refcnt_mutex);
            conn->refCount--;
-           MUTEX_EXIT(&conn->conn_data_lock);
+            MUTEX_EXIT(&rx_refcnt_mutex);
            return np;
        }
 
@@ -3018,15 +3109,15 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
         * isn't a current call, then no packet is relevant. */
        if (!call || (np->header.callNumber != currentCallNumber)) {
             if (rx_stats_active)
-                rx_MutexIncrement(rx_stats.spuriousPacketsRead, rx_stats_mutex);
+                rx_atomic_inc(&rx_stats.spuriousPacketsRead);
 #ifdef RX_ENABLE_LOCKS
            if (call) {
                MUTEX_EXIT(&call->lock);
            }
 #endif
-           MUTEX_ENTER(&conn->conn_data_lock);
+            MUTEX_ENTER(&rx_refcnt_mutex);
            conn->refCount--;
-           MUTEX_EXIT(&conn->conn_data_lock);
+            MUTEX_EXIT(&rx_refcnt_mutex);
            return np;
        }
        /* If the service security object index stamped in the packet does not
@@ -3035,9 +3126,9 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
 #ifdef RX_ENABLE_LOCKS
            MUTEX_EXIT(&call->lock);
 #endif
-           MUTEX_ENTER(&conn->conn_data_lock);
+            MUTEX_ENTER(&rx_refcnt_mutex);
            conn->refCount--;
-           MUTEX_EXIT(&conn->conn_data_lock);
+            MUTEX_EXIT(&rx_refcnt_mutex);
            return np;
        }
 
@@ -3058,7 +3149,9 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
 #ifdef RX_ENABLE_LOCKS
                rxi_SetAcksInTransmitQueue(call);
 #else
+                MUTEX_ENTER(&rx_refcnt_mutex);
                conn->refCount--;
+                MUTEX_EXIT(&rx_refcnt_mutex);
                return np;      /* xmitting; drop packet */
 #endif
            } else {
@@ -3084,11 +3177,11 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
                 * XXX code in receiveackpacket.  */
                if (ntohl(rx_GetInt32(np, FIRSTACKOFFSET)) < call->tfirst) {
                     if (rx_stats_active)
-                        rx_MutexIncrement(rx_stats.spuriousPacketsRead, rx_stats_mutex);
+                        rx_atomic_inc(&rx_stats.spuriousPacketsRead);
                    MUTEX_EXIT(&call->lock);
-                   MUTEX_ENTER(&conn->conn_data_lock);
+                    MUTEX_ENTER(&rx_refcnt_mutex);
                    conn->refCount--;
-                   MUTEX_EXIT(&conn->conn_data_lock);
+                    MUTEX_EXIT(&rx_refcnt_mutex);
                    return np;
                }
            }
@@ -3150,9 +3243,9 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
        dpf(("rxi_ReceivePacket ABORT rx_DataOf = %d", errdata));
        rxi_CallError(call, errdata);
        MUTEX_EXIT(&call->lock);
-       MUTEX_ENTER(&conn->conn_data_lock);
+        MUTEX_ENTER(&rx_refcnt_mutex);
        conn->refCount--;
-       MUTEX_EXIT(&conn->conn_data_lock);
+        MUTEX_EXIT(&rx_refcnt_mutex);
        return np;              /* xmitting; drop packet */
     }
     case RX_PACKET_TYPE_BUSY:
@@ -3177,9 +3270,9 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
            break;
 #else /* RX_ENABLE_LOCKS */
            MUTEX_EXIT(&call->lock);
-           MUTEX_ENTER(&conn->conn_data_lock);
+            MUTEX_ENTER(&rx_refcnt_mutex);
            conn->refCount--;
-           MUTEX_EXIT(&conn->conn_data_lock);
+            MUTEX_EXIT(&rx_refcnt_mutex);
            return np;          /* xmitting; drop packet */
 #endif /* RX_ENABLE_LOCKS */
        }
@@ -3200,9 +3293,9 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
      * (if not, then the time won't actually be re-evaluated here). */
     call->lastReceiveTime = clock_Sec();
     MUTEX_EXIT(&call->lock);
-    MUTEX_ENTER(&conn->conn_data_lock);
+    MUTEX_ENTER(&rx_refcnt_mutex);
     conn->refCount--;
-    MUTEX_EXIT(&conn->conn_data_lock);
+    MUTEX_EXIT(&rx_refcnt_mutex);
     return np;
 }
 
@@ -3269,8 +3362,11 @@ rxi_CheckReachEvent(struct rxevent *event, void *arg1, void *arg2)
     MUTEX_ENTER(&conn->conn_data_lock);
     conn->checkReachEvent = NULL;
     waiting = conn->flags & RX_CONN_ATTACHWAIT;
-    if (event)
+    if (event) {
+        MUTEX_ENTER(&rx_refcnt_mutex);
        conn->refCount--;
+        MUTEX_EXIT(&rx_refcnt_mutex);
+    }
     MUTEX_EXIT(&conn->conn_data_lock);
 
     if (waiting) {
@@ -3307,7 +3403,9 @@ rxi_CheckReachEvent(struct rxevent *event, void *arg1, void *arg2)
            when.sec += RX_CHECKREACH_TIMEOUT;
            MUTEX_ENTER(&conn->conn_data_lock);
            if (!conn->checkReachEvent) {
+                MUTEX_ENTER(&rx_refcnt_mutex);
                conn->refCount++;
+                MUTEX_EXIT(&rx_refcnt_mutex);
                conn->checkReachEvent =
                    rxevent_PostNow(&when, &now, rxi_CheckReachEvent, conn,
                                    NULL);
@@ -3390,7 +3488,7 @@ rxi_ReceiveDataPacket(struct rx_call *call,
     struct rx_packet *tnp;
     struct clock when, now;
     if (rx_stats_active)
-        rx_MutexIncrement(rx_stats.dataPacketsRead, rx_stats_mutex);
+        rx_atomic_inc(&rx_stats.dataPacketsRead);
 
 #ifdef KERNEL
     /* If there are no packet buffers, drop this new packet, unless we can find
@@ -3401,7 +3499,7 @@ rxi_ReceiveDataPacket(struct rx_call *call,
        rxi_NeedMorePackets = TRUE;
        MUTEX_EXIT(&rx_freePktQ_lock);
         if (rx_stats_active)
-            rx_MutexIncrement(rx_stats.noPacketBuffersOnRead, rx_stats_mutex);
+            rx_atomic_inc(&rx_stats.noPacketBuffersOnRead);
        call->rprev = np->header.serial;
        rxi_calltrace(RX_TRACE_DROP, call);
        dpf(("packet %"AFS_PTR_FMT" dropped on receipt - quota problems", np));
@@ -3414,7 +3512,10 @@ rxi_ReceiveDataPacket(struct rx_call *call,
            || clock_Gt(&call->delayedAckEvent->eventTime, &when)) {
            rxevent_Cancel(call->delayedAckEvent, call,
                           RX_CALL_REFCOUNT_DELAY);
+            MUTEX_ENTER(&rx_refcnt_mutex);
            CALL_HOLD(call, RX_CALL_REFCOUNT_DELAY);
+            MUTEX_EXIT(&rx_refcnt_mutex);
+
            call->delayedAckEvent =
                rxevent_PostNow(&when, &now, rxi_SendDelayedAck, call, 0);
        }
@@ -3467,7 +3568,7 @@ rxi_ReceiveDataPacket(struct rx_call *call,
            if (queue_IsNotEmpty(&call->rq)
                && queue_First(&call->rq, rx_packet)->header.seq == seq) {
                 if (rx_stats_active)
-                    rx_MutexIncrement(rx_stats.dupPacketsRead, rx_stats_mutex);
+                    rx_atomic_inc(&rx_stats.dupPacketsRead);
                dpf(("packet %"AFS_PTR_FMT" dropped on receipt - duplicate", np));
                rxevent_Cancel(call->delayedAckEvent, call,
                               RX_CALL_REFCOUNT_DELAY);
@@ -3558,7 +3659,7 @@ rxi_ReceiveDataPacket(struct rx_call *call,
             * application already, then this is a duplicate */
            if (seq < call->rnext) {
                 if (rx_stats_active)
-                    rx_MutexIncrement(rx_stats.dupPacketsRead, rx_stats_mutex);
+                    rx_atomic_inc(&rx_stats.dupPacketsRead);
                rxevent_Cancel(call->delayedAckEvent, call,
                               RX_CALL_REFCOUNT_DELAY);
                np = rxi_SendAck(call, np, serial, RX_ACK_DUPLICATE, istack);
@@ -3586,7 +3687,7 @@ rxi_ReceiveDataPacket(struct rx_call *call,
                /*Check for duplicate packet */
                if (seq == tp->header.seq) {
                     if (rx_stats_active)
-                        rx_MutexIncrement(rx_stats.dupPacketsRead, rx_stats_mutex);
+                        rx_atomic_inc(&rx_stats.dupPacketsRead);
                    rxevent_Cancel(call->delayedAckEvent, call,
                                   RX_CALL_REFCOUNT_DELAY);
                    np = rxi_SendAck(call, np, serial, RX_ACK_DUPLICATE,
@@ -3695,8 +3796,17 @@ rxi_ReceiveDataPacket(struct rx_call *call,
      * Send an ack when requested by the peer, or once every
      * rxi_SoftAckRate packets until the last packet has been
      * received. Always send a soft ack for the last packet in
-     * the server's reply. */
-    if (ackNeeded) {
+     * the server's reply.
+     *
+     * If we have received all of the packets for the call
+     * immediately send an RX_PACKET_TYPE_ACKALL packet so that
+     * the peer can empty its packet queue and cancel all resend
+     * events.
+     */
+    if (call->flags & RX_CALL_RECEIVE_DONE) {
+        rxevent_Cancel(call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
+        rxi_AckAll(NULL, call, 0);
+    } else if (ackNeeded) {
        rxevent_Cancel(call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
        np = rxi_SendAck(call, np, serial, ackNeeded, istack);
     } else if (call->nSoftAcks > (u_short) rxi_SoftAckRate) {
@@ -3714,12 +3824,12 @@ rxi_ReceiveDataPacket(struct rx_call *call,
            || clock_Gt(&call->delayedAckEvent->eventTime, &when)) {
            rxevent_Cancel(call->delayedAckEvent, call,
                           RX_CALL_REFCOUNT_DELAY);
+            MUTEX_ENTER(&rx_refcnt_mutex);
            CALL_HOLD(call, RX_CALL_REFCOUNT_DELAY);
+            MUTEX_EXIT(&rx_refcnt_mutex);
            call->delayedAckEvent =
                rxevent_PostNow(&when, &now, rxi_SendDelayedAck, call, 0);
        }
-    } else if (call->flags & RX_CALL_RECEIVE_DONE) {
-       rxevent_Cancel(call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
     }
 
     return np;
@@ -3799,7 +3909,8 @@ rx_ack_reason(int reason)
  */
 static void
 rxi_ComputePeerNetStats(struct rx_call *call, struct rx_packet *p,
-                       struct rx_ackPacket *ap, struct rx_packet *np)
+                       struct rx_ackPacket *ap, struct rx_packet *np,
+                       struct clock *now)
 {
     struct rx_peer *peer = call->conn->peer;
 
@@ -3808,7 +3919,7 @@ rxi_ComputePeerNetStats(struct rx_call *call, struct rx_packet *p,
     if (!(p->flags & RX_PKTFLAG_ACKED) &&
         ap->reason != RX_ACK_DELAY &&
         clock_Eq(&p->timeSent, &p->firstSent))
-       rxi_ComputeRoundTripTime(p, &p->timeSent, peer);
+       rxi_ComputeRoundTripTime(p, &p->timeSent, peer, now);
 #ifdef ADAPT_WINDOW
     rxi_ComputeRate(peer, call, p, np, ap->reason);
 #endif
@@ -3825,6 +3936,7 @@ rxi_ReceiveAckPacket(struct rx_call *call, struct rx_packet *np,
     struct rx_packet *nxp;     /* Next packet pointer for queue_Scan */
     struct rx_connection *conn = call->conn;
     struct rx_peer *peer = conn->peer;
+    struct clock now;          /* Current time, for RTT calculations */
     afs_uint32 first;
     afs_uint32 serial;
     /* because there are CM's that are bogus, sending weird values for this. */
@@ -3839,7 +3951,7 @@ rxi_ReceiveAckPacket(struct rx_call *call, struct rx_packet *np,
     int conn_data_locked = 0;
 
     if (rx_stats_active)
-        rx_MutexIncrement(rx_stats.ackPacketsRead, rx_stats_mutex);
+        rx_atomic_inc(&rx_stats.ackPacketsRead);
     ap = (struct rx_ackPacket *)rx_DataOf(np);
     nbytes = rx_Contiguous(np) - (int)((ap->acks) - (u_char *) ap);
     if (nbytes < 0)
@@ -3958,11 +4070,14 @@ rxi_ReceiveAckPacket(struct rx_call *call, struct rx_packet *np,
      * acknowledged as having been sent to the peer's upper level.
      * All other packets must be retained.  So only packets with
      * sequence numbers < ap->firstPacket are candidates. */
+
+    clock_GetTime(&now);
+
     for (queue_Scan(&call->tq, tp, nxp, rx_packet)) {
        if (tp->header.seq >= first)
            break;
        call->tfirst = tp->header.seq + 1;
-        rxi_ComputePeerNetStats(call, tp, ap, np);
+        rxi_ComputePeerNetStats(call, tp, ap, np, &now);
        if (!(tp->flags & RX_PKTFLAG_ACKED)) {
            newAckCount++;
        }
@@ -4024,7 +4139,7 @@ rxi_ReceiveAckPacket(struct rx_call *call, struct rx_packet *np,
        if (tp->header.seq >= first)
 #endif /* RX_ENABLE_LOCKS */
 #endif /* AFS_GLOBAL_RXLOCK_KERNEL */
-            rxi_ComputePeerNetStats(call, tp, ap, np);
+            rxi_ComputePeerNetStats(call, tp, ap, np, &now);
 
        /* Set the acknowledge flag per packet based on the
         * information in the ack packet. An acknowlegded packet can
@@ -4456,10 +4571,8 @@ rxi_AttachServerProc(struct rx_call *call,
 
        if (!(call->flags & RX_CALL_WAIT_PROC)) {
            call->flags |= RX_CALL_WAIT_PROC;
-            MUTEX_ENTER(&rx_waiting_mutex);
-            rx_nWaiting++;
-            rx_nWaited++;
-            MUTEX_EXIT(&rx_waiting_mutex);
+           rx_atomic_inc(&rx_nWaiting);
+           rx_atomic_inc(&rx_nWaited);
            rxi_calltrace(RX_CALL_ARRIVAL, call);
            SET_CALL_QUEUE_LOCK(call, &rx_serverPool_lock);
            queue_Append(&rx_incomingCallQueue, call);
@@ -4477,7 +4590,9 @@ rxi_AttachServerProc(struct rx_call *call,
            *tnop = sq->tno;
            *sq->socketp = socket;
            clock_GetTime(&call->startTime);
+            MUTEX_ENTER(&rx_refcnt_mutex);
            CALL_HOLD(call, RX_CALL_REFCOUNT_BEGIN);
+            MUTEX_EXIT(&rx_refcnt_mutex);
        } else {
            sq->newcall = call;
        }
@@ -4487,9 +4602,7 @@ rxi_AttachServerProc(struct rx_call *call,
            if (queue_IsOnQueue(call)) {
                queue_Remove(call);
 
-                MUTEX_ENTER(&rx_waiting_mutex);
-                rx_nWaiting--;
-                MUTEX_EXIT(&rx_waiting_mutex);
+               rx_atomic_dec(&rx_nWaiting);
            }
        }
        call->state = RX_STATE_ACTIVE;
@@ -4537,7 +4650,9 @@ rxi_AckAll(struct rxevent *event, struct rx_call *call, char *dummy)
     if (event) {
        MUTEX_ENTER(&call->lock);
        call->delayedAckEvent = NULL;
+        MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_RELE(call, RX_CALL_REFCOUNT_ACKALL);
+        MUTEX_EXIT(&rx_refcnt_mutex);
     }
     rxi_SendSpecial(call, call->conn, (struct rx_packet *)0,
                    RX_PACKET_TYPE_ACKALL, NULL, 0, 0);
@@ -4560,7 +4675,9 @@ rxi_SendDelayedAck(struct rxevent *event, void *arg1, void *unused)
        MUTEX_ENTER(&call->lock);
        if (event == call->delayedAckEvent)
            call->delayedAckEvent = NULL;
+        MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_RELE(call, RX_CALL_REFCOUNT_DELAY);
+        MUTEX_EXIT(&rx_refcnt_mutex);
     }
     (void)rxi_SendAck(call, 0, 0, RX_ACK_DELAY, 0);
     if (event)
@@ -4712,7 +4829,9 @@ rxi_SendCallAbort(struct rx_call *call, struct rx_packet *packet,
        clock_GetTime(&now);
        when = now;
        clock_Addmsec(&when, rxi_callAbortDelay);
+        MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_HOLD(call, RX_CALL_REFCOUNT_ABORT);
+        MUTEX_EXIT(&rx_refcnt_mutex);
        call->delayedAbortEvent =
            rxevent_PostNow(&when, &now, rxi_SendDelayedCallAbort, call, 0);
     }
@@ -4788,7 +4907,9 @@ rxi_ConnectionError(struct rx_connection *conn,
            rxevent_Cancel(conn->checkReachEvent, (struct rx_call *)0, 0);
            conn->checkReachEvent = 0;
            conn->flags &= ~RX_CONN_ATTACHWAIT;
+            MUTEX_ENTER(&rx_refcnt_mutex);
            conn->refCount--;
+            MUTEX_EXIT(&rx_refcnt_mutex);
        }
        MUTEX_EXIT(&conn->conn_data_lock);
        for (i = 0; i < RX_MAXCALLS; i++) {
@@ -4801,7 +4922,7 @@ rxi_ConnectionError(struct rx_connection *conn,
        }
        conn->error = error;
         if (rx_stats_active)
-            rx_MutexIncrement(rx_stats.fatalErrors, rx_stats_mutex);
+            rx_atomic_inc(&rx_stats.fatalErrors);
     }
 }
 
@@ -4823,7 +4944,6 @@ rxi_CallError(struct rx_call *call, afs_int32 error)
     rxi_ResetCall(call, 0);
 #endif
     call->error = error;
-    call->mode = RX_MODE_ERROR;
 }
 
 /* Reset various fields in a call structure, and wakeup waiting
@@ -4958,10 +5078,7 @@ rxi_ResetCall(struct rx_call *call, int newcall)
        if (queue_IsOnQueue(call)) {
            queue_Remove(call);
            if (flags & RX_CALL_WAIT_PROC) {
-
-                MUTEX_ENTER(&rx_waiting_mutex);
-                rx_nWaiting--;
-                MUTEX_EXIT(&rx_waiting_mutex);
+               rx_atomic_dec(&rx_nWaiting);
            }
        }
        MUTEX_EXIT(call->call_queue_lock);
@@ -4971,7 +5088,7 @@ rxi_ResetCall(struct rx_call *call, int newcall)
     if (queue_IsOnQueue(call)) {
        queue_Remove(call);
        if (flags & RX_CALL_WAIT_PROC)
-           rx_nWaiting--;
+           rx_atomic_dec(&rx_nWaiting);
     }
 #endif /* RX_ENABLE_LOCKS */
 
@@ -5246,7 +5363,7 @@ rxi_SendAck(struct rx_call *call,
        }
     }
     if (rx_stats_active)
-        rx_MutexIncrement(rx_stats.ackPacketsSent, rx_stats_mutex);
+        rx_atomic_inc(&rx_stats.ackPacketsSent);
 #ifndef RX_ENABLE_TSFPQ
     if (!optionalPacket)
        rxi_FreePacket(p);
@@ -5274,9 +5391,9 @@ rxi_SendList(struct rx_call *call, struct rx_packet **list, int len,
 
     if (rx_stats_active) {
         if (resending)
-            rx_MutexAdd(rx_stats.dataPacketsReSent, len, rx_stats_mutex);
+            rx_atomic_add(&rx_stats.dataPacketsReSent, len);
         else
-            rx_MutexAdd(rx_stats.dataPacketsSent, len, rx_stats_mutex);
+            rx_atomic_add(&rx_stats.dataPacketsSent, len);
     }
 
     if (list[len - 1]->header.flags & RX_LAST_PACKET) {
@@ -5340,15 +5457,19 @@ rxi_SendList(struct rx_call *call, struct rx_packet **list, int len,
      * safe to nuke any scheduled end-of-packets ack */
     rxevent_Cancel(call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
 
-    CALL_HOLD(call, RX_CALL_REFCOUNT_SEND);
     MUTEX_EXIT(&call->lock);
+    MUTEX_ENTER(&rx_refcnt_mutex);
+    CALL_HOLD(call, RX_CALL_REFCOUNT_SEND);
+    MUTEX_EXIT(&rx_refcnt_mutex);
     if (len > 1) {
        rxi_SendPacketList(call, conn, list, len, istack);
     } else {
        rxi_SendPacket(call, conn, list[0], istack);
     }
     MUTEX_ENTER(&call->lock);
+    MUTEX_ENTER(&rx_refcnt_mutex);
     CALL_RELE(call, RX_CALL_REFCOUNT_SEND);
+    MUTEX_EXIT(&rx_refcnt_mutex);
 
     /* Update last send time for this call (for keep-alive
      * processing), and for the connection (so that we can discover
@@ -5489,7 +5610,6 @@ rxi_Start(struct rxevent *event,
     int haveEvent;
     int nXmitPackets;
     int maxXmitPackets;
-    struct rx_packet **xmitList;
     int resending = 0;
 
     /* If rxi_Start is being called as a result of a resend event,
@@ -5497,7 +5617,9 @@ rxi_Start(struct rxevent *event,
      * structure, since there is no longer a per-call retransmission
      * event pending. */
     if (event && event == call->resendEvent) {
+        MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_RELE(call, RX_CALL_REFCOUNT_RESEND);
+        MUTEX_EXIT(&rx_refcnt_mutex);
        call->resendEvent = NULL;
        resending = 1;
        if (queue_IsEmpty(&call->tq)) {
@@ -5546,7 +5668,7 @@ rxi_Start(struct rxevent *event,
     if (call->error) {
 #ifdef AFS_GLOBAL_RXLOCK_KERNEL
         if (rx_stats_active)
-            rx_MutexIncrement(rx_tq_debug.rxi_start_in_error, rx_stats_mutex);
+            rx_atomic_inc(&rx_tq_debug.rxi_start_in_error);
 #endif
        return;
     }
@@ -5591,15 +5713,6 @@ rxi_Start(struct rxevent *event,
 #endif /* AFS_GLOBAL_RXLOCK_KERNEL */
                nXmitPackets = 0;
                maxXmitPackets = MIN(call->twind, call->cwind);
-               xmitList = (struct rx_packet **)
-#if defined(KERNEL) && !defined(UKERNEL) && defined(AFS_FBSD80_ENV)
-                   /* XXXX else we must drop any mtx we hold */
-                   afs_osi_Alloc_NoSleep(maxXmitPackets * sizeof(struct rx_packet *));
-#else
-               osi_Alloc(maxXmitPackets * sizeof(struct rx_packet *));
-#endif
-               if (xmitList == NULL)
-                   osi_Panic("rxi_Start, failed to allocate xmit list");
                for (queue_Scan(&call->tq, p, nxp, rx_packet)) {
                    if (call->flags & RX_CALL_FAST_RECOVER_WAIT) {
                        /* We shouldn't be sending packets if a thread is waiting
@@ -5628,7 +5741,7 @@ rxi_Start(struct rxevent *event,
                        /* Since we may block, don't trust this */
                        usenow.sec = usenow.usec = 0;
                         if (rx_stats_active)
-                            rx_MutexIncrement(rx_stats.ignoreAckedPacket, rx_stats_mutex);
+                            rx_atomic_inc(&rx_stats.ignoreAckedPacket);
                        continue;       /* Ignore this packet if it has been acknowledged */
                    }
 
@@ -5653,11 +5766,9 @@ rxi_Start(struct rxevent *event,
                    /* Transmit the packet if it needs to be sent. */
                    if (!clock_Lt(&now, &p->retryTime)) {
                        if (nXmitPackets == maxXmitPackets) {
-                           rxi_SendXmitList(call, xmitList, nXmitPackets,
-                                            istack, &now, &retryTime,
-                                            resending);
-                           osi_Free(xmitList, maxXmitPackets *
-                                    sizeof(struct rx_packet *));
+                           rxi_SendXmitList(call, call->xmitList,
+                                            nXmitPackets, istack, &now, 
+                                            &retryTime, resending);
                            goto restart;
                        }
                         dpf(("call %d xmit packet %"AFS_PTR_FMT" now %u.%06u retryTime %u.%06u nextRetry %u.%06u\n",
@@ -5665,18 +5776,16 @@ rxi_Start(struct rxevent *event,
                               now.sec, now.usec,
                               p->retryTime.sec, p->retryTime.usec,
                               retryTime.sec, retryTime.usec));
-                       xmitList[nXmitPackets++] = p;
+                       call->xmitList[nXmitPackets++] = p;
                    }
                }
 
                /* xmitList now hold pointers to all of the packets that are
                 * ready to send. Now we loop to send the packets */
                if (nXmitPackets > 0) {
-                   rxi_SendXmitList(call, xmitList, nXmitPackets, istack,
-                                    &now, &retryTime, resending);
+                   rxi_SendXmitList(call, call->xmitList, nXmitPackets,
+                                    istack, &now, &retryTime, resending);
                }
-               osi_Free(xmitList,
-                        maxXmitPackets * sizeof(struct rx_packet *));
 
 #ifdef AFS_GLOBAL_RXLOCK_KERNEL
                /*
@@ -5703,7 +5812,7 @@ rxi_Start(struct rxevent *event,
                     * process that the call is in an error state.
                     */
                     if (rx_stats_active)
-                        rx_MutexIncrement(rx_tq_debug.rxi_start_aborted, rx_stats_mutex);
+                        rx_atomic_inc(&rx_tq_debug.rxi_start_aborted);
                    call->flags &= ~RX_CALL_TQ_BUSY;
                    if (call->tqWaiters || (call->flags & RX_CALL_TQ_WAIT)) {
                        dpf(("call error %d while xmit %p has %d waiters and flags %d\n",
@@ -5782,7 +5891,9 @@ rxi_Start(struct rxevent *event,
                    /* Post a new event to re-run rxi_Start when retries may be needed */
                    if (haveEvent && !(call->flags & RX_CALL_NEED_START)) {
 #ifdef RX_ENABLE_LOCKS
+                        MUTEX_ENTER(&rx_refcnt_mutex);
                        CALL_HOLD(call, RX_CALL_REFCOUNT_RESEND);
+                        MUTEX_EXIT(&rx_refcnt_mutex);
                        call->resendEvent =
                            rxevent_PostNow2(&retryTime, &usenow,
                                             rxi_StartUnlocked,
@@ -5843,11 +5954,15 @@ rxi_Send(struct rx_call *call, struct rx_packet *p,
     rxevent_Cancel(call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
 
     /* Actually send the packet, filling in more connection-specific fields */
-    CALL_HOLD(call, RX_CALL_REFCOUNT_SEND);
     MUTEX_EXIT(&call->lock);
+    MUTEX_ENTER(&rx_refcnt_mutex);
+    CALL_HOLD(call, RX_CALL_REFCOUNT_SEND);
+    MUTEX_EXIT(&rx_refcnt_mutex);
     rxi_SendPacket(call, conn, p, istack);
-    MUTEX_ENTER(&call->lock);
+    MUTEX_ENTER(&rx_refcnt_mutex);
     CALL_RELE(call, RX_CALL_REFCOUNT_SEND);
+    MUTEX_EXIT(&rx_refcnt_mutex);
+    MUTEX_ENTER(&call->lock);
 
     /* Update last send time for this call (for keep-alive
      * processing), and for the connection (so that we can discover
@@ -5883,7 +5998,8 @@ rxi_CheckCall(struct rx_call *call)
 {
     struct rx_connection *conn = call->conn;
     afs_uint32 now;
-    afs_uint32 deadTime;
+    afs_uint32 deadTime, idleDeadTime = 0, hardDeadTime = 0;
+    afs_uint32 fudgeFactor;
     int cerror = 0;
     int newmtu = 0;
 
@@ -5895,11 +6011,11 @@ rxi_CheckCall(struct rx_call *call)
        return 0;
     }
 #endif
-    /* dead time + RTT + 8*MDEV, rounded up to next second. */
-    deadTime =
-       (((afs_uint32) conn->secondsUntilDead << 10) +
-        ((afs_uint32) conn->peer->rtt >> 3) +
-        ((afs_uint32) conn->peer->rtt_dev << 1) + 1023) >> 10;
+    /* RTT + 8*MDEV, rounded up to the next second. */
+    fudgeFactor = (((afs_uint32) conn->peer->rtt >> 3) +
+                   ((afs_uint32) conn->peer->rtt_dev << 1) + 1023) >> 10;
+
+    deadTime = conn->secondsUntilDead + fudgeFactor;
     now = clock_Sec();
     /* These are computed to the second (+- 1 second).  But that's
      * good enough for these values, which should be a significant
@@ -5943,13 +6059,16 @@ rxi_CheckCall(struct rx_call *call)
            rxevent_Cancel(call->resendEvent, call, RX_CALL_REFCOUNT_RESEND);
            rxevent_Cancel(call->keepAliveEvent, call,
                           RX_CALL_REFCOUNT_ALIVE);
+            MUTEX_ENTER(&rx_refcnt_mutex);
            if (call->refCount == 0) {
                rxi_FreeCall(call, haveCTLock);
+                MUTEX_EXIT(&rx_refcnt_mutex);
                return -2;
            }
+            MUTEX_EXIT(&rx_refcnt_mutex);
            return -1;
 #else /* RX_ENABLE_LOCKS */
-           rxi_FreeCall(call);
+           rxi_FreeCall(call, 0);
            return -2;
 #endif /* RX_ENABLE_LOCKS */
        }
@@ -5957,36 +6076,47 @@ rxi_CheckCall(struct rx_call *call)
         * to pings; active calls are simply flagged in error, so the
         * attached process can die reasonably gracefully. */
     }
+
+    if (conn->idleDeadTime) {
+       idleDeadTime = conn->idleDeadTime + fudgeFactor;
+    }
+
     /* see if we have a non-activity timeout */
-    if (call->startWait && conn->idleDeadTime
-       && ((call->startWait + conn->idleDeadTime) < now) &&
+    if (call->startWait && idleDeadTime
+       && ((call->startWait + idleDeadTime) < now) &&
        (call->flags & RX_CALL_READER_WAIT)) {
        if (call->state == RX_STATE_ACTIVE) {
            cerror = RX_CALL_TIMEOUT;
            goto mtuout;
        }
     }
-    if (call->lastSendData && conn->idleDeadTime && (conn->idleDeadErr != 0)
-        && ((call->lastSendData + conn->idleDeadTime) < now)) {
+    if (call->lastSendData && idleDeadTime && (conn->idleDeadErr != 0)
+        && ((call->lastSendData + idleDeadTime) < now)) {
        if (call->state == RX_STATE_ACTIVE) {
            cerror = conn->idleDeadErr;
            goto mtuout;
        }
     }
+
+    if (hardDeadTime) {
+       hardDeadTime = conn->hardDeadTime + fudgeFactor;
+    }
+
     /* see if we have a hard timeout */
-    if (conn->hardDeadTime
-       && (now > (conn->hardDeadTime + call->startTime.sec))) {
+    if (hardDeadTime
+       && (now > (hardDeadTime + call->startTime.sec))) {
        if (call->state == RX_STATE_ACTIVE)
            rxi_CallError(call, RX_CALL_TIMEOUT);
        return -1;
     }
     return 0;
 mtuout:
-    if (conn->msgsizeRetryErr && cerror != RX_CALL_TIMEOUT) {
+    if (conn->msgsizeRetryErr && cerror != RX_CALL_TIMEOUT
+       && call->lastReceiveTime) {
        int oldMTU = conn->peer->ifMTU;
 
        /* if we thought we could send more, perhaps things got worse */
-       if (call->conn->peer->maxPacketSize > conn->lastPacketSize)
+       if (conn->peer->maxPacketSize > conn->lastPacketSize)
            /* maxpacketsize will be cleared in rxi_SetPeerMtu */
            newmtu = MAX(conn->peer->maxPacketSize-RX_IPUDP_SIZE,
                         conn->lastPacketSize-(128+RX_IPUDP_SIZE));
@@ -6051,14 +6181,17 @@ rxi_NatKeepAliveEvent(struct rxevent *event, void *arg1, void *dummy)
     osi_NetSend(socket, &taddr, tmpiov, 1, 1 + sizeof(struct rx_header), 1);
 
     MUTEX_ENTER(&conn->conn_data_lock);
+    MUTEX_ENTER(&rx_refcnt_mutex);
     /* Only reschedule ourselves if the connection would not be destroyed */
     if (conn->refCount <= 1) {
        conn->natKeepAliveEvent = NULL;
+        MUTEX_EXIT(&rx_refcnt_mutex);
        MUTEX_EXIT(&conn->conn_data_lock);
        rx_DestroyConnection(conn); /* drop the reference for this */
     } else {
-       conn->natKeepAliveEvent = NULL;
        conn->refCount--; /* drop the reference for this */
+        MUTEX_EXIT(&rx_refcnt_mutex);
+       conn->natKeepAliveEvent = NULL;
        rxi_ScheduleNatKeepAliveEvent(conn);
        MUTEX_EXIT(&conn->conn_data_lock);
     }
@@ -6072,7 +6205,9 @@ rxi_ScheduleNatKeepAliveEvent(struct rx_connection *conn)
        clock_GetTime(&now);
        when = now;
        when.sec += conn->secondsUntilNatPing;
+        MUTEX_ENTER(&rx_refcnt_mutex);
        conn->refCount++; /* hold a reference for this */
+        MUTEX_EXIT(&rx_refcnt_mutex);
        conn->natKeepAliveEvent =
            rxevent_PostNow(&when, &now, rxi_NatKeepAliveEvent, conn, 0);
     }
@@ -6109,8 +6244,10 @@ rxi_KeepAliveEvent(struct rxevent *event, void *arg1, void *dummy)
     struct rx_connection *conn;
     afs_uint32 now;
 
-    MUTEX_ENTER(&call->lock);
+    MUTEX_ENTER(&rx_refcnt_mutex);
     CALL_RELE(call, RX_CALL_REFCOUNT_ALIVE);
+    MUTEX_EXIT(&rx_refcnt_mutex);
+    MUTEX_ENTER(&call->lock);
     if (event == call->keepAliveEvent)
        call->keepAliveEvent = NULL;
     now = clock_Sec();
@@ -6149,8 +6286,11 @@ rxi_GrowMTUEvent(struct rxevent *event, void *arg1, void *dummy)
     struct rx_call *call = arg1;
     struct rx_connection *conn;
 
-    MUTEX_ENTER(&call->lock);
+    MUTEX_ENTER(&rx_refcnt_mutex);
     CALL_RELE(call, RX_CALL_REFCOUNT_ALIVE);
+    MUTEX_EXIT(&rx_refcnt_mutex);
+    MUTEX_ENTER(&call->lock);
+
     if (event == call->growMTUEvent)
        call->growMTUEvent = NULL;
 
@@ -6192,7 +6332,9 @@ rxi_ScheduleKeepAliveEvent(struct rx_call *call)
        clock_GetTime(&now);
        when = now;
        when.sec += call->conn->secondsUntilPing;
+        MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_HOLD(call, RX_CALL_REFCOUNT_ALIVE);
+        MUTEX_EXIT(&rx_refcnt_mutex);
        call->keepAliveEvent =
            rxevent_PostNow(&when, &now, rxi_KeepAliveEvent, call, 0);
     }
@@ -6215,7 +6357,9 @@ rxi_ScheduleGrowMTUEvent(struct rx_call *call, int secs)
        }
 
        when.sec += secs;
+        MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_HOLD(call, RX_CALL_REFCOUNT_ALIVE);
+        MUTEX_EXIT(&rx_refcnt_mutex);
        call->growMTUEvent =
            rxevent_PostNow(&when, &now, rxi_GrowMTUEvent, call, 0);
     }
@@ -6292,8 +6436,10 @@ rxi_SendDelayedCallAbort(struct rxevent *event,
                            (char *)&error, sizeof(error), 0);
        rxi_FreePacket(packet);
     }
-    CALL_RELE(call, RX_CALL_REFCOUNT_ABORT);
     MUTEX_EXIT(&call->lock);
+    MUTEX_ENTER(&rx_refcnt_mutex);
+    CALL_RELE(call, RX_CALL_REFCOUNT_ABORT);
+    MUTEX_EXIT(&rx_refcnt_mutex);
 }
 
 /* This routine is called periodically (every RX_AUTH_REQUEST_TIMEOUT
@@ -6372,21 +6518,20 @@ rxi_ChallengeOn(struct rx_connection *conn)
 
 /* rxi_ComputeRoundTripTime is called with peer locked. */
 /* sentp and/or peer may be null */
-void
+static void
 rxi_ComputeRoundTripTime(struct rx_packet *p,
                         struct clock *sentp,
-                        struct rx_peer *peer)
+                        struct rx_peer *peer,
+                        struct clock *now)
 {
     struct clock thisRtt, *rttp = &thisRtt;
-
     int rtt_timeout;
 
-    clock_GetTime(rttp);
+    thisRtt = *now;
 
-    if (clock_Lt(rttp, sentp)) {
-       clock_Zero(rttp);
+    if (clock_Lt(rttp, sentp))
        return;                 /* somebody set the clock back, don't count this time. */
-    }
+
     clock_Sub(rttp, sentp);
     dpf(("rxi_ComputeRoundTripTime(call=%d packet=%"AFS_PTR_FMT" rttp=%d.%06d sec)\n",
           p->header.callNumber, p, rttp->sec, rttp->usec));
@@ -6412,7 +6557,7 @@ rxi_ComputeRoundTripTime(struct rx_packet *p,
             rx_stats.maxRtt = *rttp;
         }
         clock_Add(&rx_stats.totalRtt, rttp);
-        rx_stats.nRttSamples++;
+        rx_atomic_inc(&rx_stats.nRttSamples);
         MUTEX_EXIT(&rx_stats_mutex);
     }
 
@@ -6539,10 +6684,12 @@ rxi_ReapConnections(struct rxevent *unused, void *unused1, void *unused2)
                    /* This only actually destroys the connection if
                     * there are no outstanding calls */
                    MUTEX_ENTER(&conn->conn_data_lock);
+                    MUTEX_ENTER(&rx_refcnt_mutex);
                    if (!havecalls && !conn->refCount
                        && ((conn->lastSendTime + rx_idleConnectionTime) <
                            now.sec)) {
                        conn->refCount++;       /* it will be decr in rx_DestroyConn */
+                        MUTEX_EXIT(&rx_refcnt_mutex);
                        MUTEX_EXIT(&conn->conn_data_lock);
 #ifdef RX_ENABLE_LOCKS
                        rxi_DestroyConnectionNoLock(conn);
@@ -6552,6 +6699,7 @@ rxi_ReapConnections(struct rxevent *unused, void *unused1, void *unused2)
                    }
 #ifdef RX_ENABLE_LOCKS
                    else {
+                        MUTEX_EXIT(&rx_refcnt_mutex);
                        MUTEX_EXIT(&conn->conn_data_lock);
                    }
 #endif /* RX_ENABLE_LOCKS */
@@ -6617,7 +6765,7 @@ rxi_ReapConnections(struct rxevent *unused, void *unused1, void *unused2)
                        prev->next = next;
 
                     if (rx_stats_active)
-                        rx_MutexDecrement(rx_stats.nPeerStructs, rx_stats_mutex);
+                        rx_atomic_dec(&rx_stats.nPeerStructs);
 
                     /*
                      * Now if we hold references on 'prev' and 'next'
@@ -7047,7 +7195,8 @@ void
 rx_PrintStats(FILE * file)
 {
     MUTEX_ENTER(&rx_stats_mutex);
-    rx_PrintTheseStats(file, &rx_stats, sizeof(rx_stats), rx_nFreePackets,
+    rx_PrintTheseStats(file, (struct rx_statistics *) &rx_stats,
+                      sizeof(rx_stats), rx_nFreePackets,
                       RX_DEBUGI_VERSION);
     MUTEX_EXIT(&rx_stats_mutex);
 }
@@ -7611,7 +7760,7 @@ shutdown_rx(void)
                next = peer->next;
                rxi_FreePeer(peer);
                 if (rx_stats_active)
-                    rx_MutexDecrement(rx_stats.nPeerStructs, rx_stats_mutex);
+                    rx_atomic_dec(&rx_stats.nPeerStructs);
            }
             MUTEX_EXIT(&rx_peerHashTable_lock);
        }