rx: Correctly test for end of call queue
[openafs.git] / src / rx / rx.c
index cd1aa75..e3a4c6a 100644 (file)
@@ -78,6 +78,11 @@ extern afs_int32 afs_termState;
 #include "rx_trace.h"
 #include "rx_internal.h"
 #include "rx_stats.h"
+#include "rx_event.h"
+
+#include "rx_conn.h"
+#include "rx_call.h"
+#include "rx_packet.h"
 
 #include <afs/rxgen_consts.h>
 
@@ -181,6 +186,13 @@ afs_kmutex_t rx_atomic_mutex;
 /* Forward prototypes */
 static struct rx_call * rxi_NewCall(struct rx_connection *, int);
 
+static_inline void
+putConnection (struct rx_connection *conn) {
+    MUTEX_ENTER(&rx_refcnt_mutex);
+    conn->refCount--;
+    MUTEX_EXIT(&rx_refcnt_mutex);
+}
+
 #ifdef AFS_PTHREAD_ENV
 
 /*
@@ -196,7 +208,6 @@ extern afs_kmutex_t des_init_mutex;
 extern afs_kmutex_t des_random_mutex;
 extern afs_kmutex_t rx_clock_mutex;
 extern afs_kmutex_t rxi_connCacheMutex;
-extern afs_kmutex_t rx_event_mutex;
 extern afs_kmutex_t event_handler_mutex;
 extern afs_kmutex_t listener_mutex;
 extern afs_kmutex_t rx_if_init_mutex;
@@ -222,7 +233,6 @@ rxi_InitPthread(void)
     MUTEX_INIT(&rx_refcnt_mutex, "refcnts", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&epoch_mutex, "epoch", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_init_mutex, "init", MUTEX_DEFAULT, 0);
-    MUTEX_INIT(&rx_event_mutex, "event", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&event_handler_mutex, "event handler", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rxi_connCacheMutex, "conn cache", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&listener_mutex, "listener", MUTEX_DEFAULT, 0);
@@ -653,9 +663,7 @@ rxi_rto_startTimer(struct rx_call *call, int lastPacket, int istack)
     if (lastPacket && call->conn->type == RX_CLIENT_CONNECTION)
        clock_Addmsec(&retryTime, 400);
 
-    MUTEX_ENTER(&rx_refcnt_mutex);
     CALL_HOLD(call, RX_CALL_REFCOUNT_RESEND);
-    MUTEX_EXIT(&rx_refcnt_mutex);
     call->resendEvent = rxevent_Post(&retryTime, &now, rxi_Resend,
                                     call, NULL, istack);
 }
@@ -776,22 +784,19 @@ rxi_PostDelayedAckEvent(struct rx_call *call, struct clock *offset)
     clock_Add(&when, offset);
 
     if (!call->delayedAckEvent
-       || clock_Gt(&call->delayedAckEvent->eventTime, &when)) {
+       || clock_Gt(&call->delayedAckTime, &when)) {
 
         rxevent_Cancel(&call->delayedAckEvent, call,
                       RX_CALL_REFCOUNT_DELAY);
-       MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_HOLD(call, RX_CALL_REFCOUNT_DELAY);
-       MUTEX_EXIT(&rx_refcnt_mutex);
 
        call->delayedAckEvent = rxevent_Post(&when, &now,
                                             rxi_SendDelayedAck,
                                             call, NULL, 0);
+       call->delayedAckTime = when;
     }
 }
 
-
-
 /* called with unincremented nRequestsRunning to see if it is OK to start
  * a new thread in this service.  Could be "no" for two reasons: over the
  * max quota, or would prevent others from reaching their min quota.
@@ -1490,9 +1495,7 @@ rx_NewCall(struct rx_connection *conn)
                          */
                         call->state = RX_STATE_RESET;
                         MUTEX_EXIT(&conn->conn_call_lock);
-                        MUTEX_ENTER(&rx_refcnt_mutex);
                         CALL_HOLD(call, RX_CALL_REFCOUNT_BEGIN);
-                        MUTEX_EXIT(&rx_refcnt_mutex);
                         rxi_ResetCall(call, 0);
                         (*call->callNumber)++;
                         if (MUTEX_TRYENTER(&conn->conn_call_lock))
@@ -1524,9 +1527,7 @@ rx_NewCall(struct rx_connection *conn)
                          * Instead, cycle through one more time to see if
                          * we can find a call that can call our own.
                          */
-                        MUTEX_ENTER(&rx_refcnt_mutex);
                         CALL_RELE(call, RX_CALL_REFCOUNT_BEGIN);
-                        MUTEX_EXIT(&rx_refcnt_mutex);
                         wait = 0;
                     }
                     MUTEX_EXIT(&call->lock);
@@ -1543,9 +1544,7 @@ rx_NewCall(struct rx_connection *conn)
 
                 /* rxi_NewCall returns with mutex locked */
                call = rxi_NewCall(conn, i);
-                MUTEX_ENTER(&rx_refcnt_mutex);
                 CALL_HOLD(call, RX_CALL_REFCOUNT_BEGIN);
-                MUTEX_EXIT(&rx_refcnt_mutex);
                break;
            }
        }
@@ -2007,7 +2006,7 @@ rx_GetCall(int tno, struct rx_service *cur_service, osi_socket * socketp)
                }
                MUTEX_ENTER(&rx_pthread_mutex);
                if (tno == rxi_fcfs_thread_num
-                   || !tcall->queue_item_header.next) {
+                       || queue_IsLast(&rx_incomingCallQueue, tcall)) {
                    MUTEX_EXIT(&rx_pthread_mutex);
                    /* If we're the fcfs thread , then  we'll just use
                     * this call. If we haven't been able to find an optimal
@@ -2126,9 +2125,7 @@ rx_GetCall(int tno, struct rx_service *cur_service, osi_socket * socketp)
             call));
 
        MUTEX_EXIT(&call->lock);
-        MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_HOLD(call, RX_CALL_REFCOUNT_BEGIN);
-        MUTEX_EXIT(&rx_refcnt_mutex);
     } else {
        dpf(("rx_GetCall(socketp=%p, *socketp=0x%x)\n", socketp, *socketp));
     }
@@ -2449,9 +2446,7 @@ rx_EndCall(struct rx_call *call, afs_int32 rc)
         rxi_FreePackets(0, &call->iovq);
     MUTEX_EXIT(&call->lock);
 
-    MUTEX_ENTER(&rx_refcnt_mutex);
     CALL_RELE(call, RX_CALL_REFCOUNT_BEGIN);
-    MUTEX_EXIT(&rx_refcnt_mutex);
     if (conn->type == RX_CLIENT_CONNECTION) {
        MUTEX_ENTER(&conn->conn_data_lock);
        conn->flags &= ~RX_CONN_BUSY;
@@ -3189,9 +3184,7 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
        MUTEX_ENTER(&conn->conn_data_lock);
        if (np->header.type != RX_PACKET_TYPE_ABORT)
            np = rxi_SendConnectionAbort(conn, np, 1, 0);
-        MUTEX_ENTER(&rx_refcnt_mutex);
-       conn->refCount--;
-        MUTEX_EXIT(&rx_refcnt_mutex);
+       putConnection(conn);
        MUTEX_EXIT(&conn->conn_data_lock);
        return np;
     }
@@ -3204,42 +3197,31 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
            afs_int32 errcode = ntohl(rx_GetInt32(np, 0));
            dpf(("rxi_ReceivePacket ABORT rx_GetInt32 = %d\n", errcode));
            rxi_ConnectionError(conn, errcode);
-            MUTEX_ENTER(&rx_refcnt_mutex);
-           conn->refCount--;
-            MUTEX_EXIT(&rx_refcnt_mutex);
+           putConnection(conn);
            return np;
        }
        case RX_PACKET_TYPE_CHALLENGE:
            tnp = rxi_ReceiveChallengePacket(conn, np, 1);
-            MUTEX_ENTER(&rx_refcnt_mutex);
-           conn->refCount--;
-            MUTEX_EXIT(&rx_refcnt_mutex);
+           putConnection(conn);
            return tnp;
        case RX_PACKET_TYPE_RESPONSE:
            tnp = rxi_ReceiveResponsePacket(conn, np, 1);
-            MUTEX_ENTER(&rx_refcnt_mutex);
-           conn->refCount--;
-            MUTEX_EXIT(&rx_refcnt_mutex);
+           putConnection(conn);
            return tnp;
        case RX_PACKET_TYPE_PARAMS:
        case RX_PACKET_TYPE_PARAMS + 1:
        case RX_PACKET_TYPE_PARAMS + 2:
            /* ignore these packet types for now */
-            MUTEX_ENTER(&rx_refcnt_mutex);
-           conn->refCount--;
-            MUTEX_EXIT(&rx_refcnt_mutex);
+           putConnection(conn);
            return np;
 
-
        default:
            /* Should not reach here, unless the peer is broken: send an
             * abort packet */
            rxi_ConnectionError(conn, RX_PROTOCOL_ERROR);
            MUTEX_ENTER(&conn->conn_data_lock);
            tnp = rxi_SendConnectionAbort(conn, np, 1, 0);
-            MUTEX_ENTER(&rx_refcnt_mutex);
-           conn->refCount--;
-            MUTEX_EXIT(&rx_refcnt_mutex);
+           putConnection(conn);
            MUTEX_EXIT(&conn->conn_data_lock);
            return tnp;
        }
@@ -3284,9 +3266,7 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
                 rxi_CallError(call, rx_BusyError);
                 tp = rxi_SendCallAbort(call, np, 1, 0);
                 MUTEX_EXIT(&call->lock);
-                MUTEX_ENTER(&rx_refcnt_mutex);
-                conn->refCount--;
-                MUTEX_EXIT(&rx_refcnt_mutex);
+               putConnection(conn);
                 if (rx_stats_active)
                     rx_atomic_inc(&rx_stats.nBusies);
                 return tp;
@@ -3301,9 +3281,7 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
          */
         if (rx_stats_active)
             rx_atomic_inc(&rx_stats.spuriousPacketsRead);
-        MUTEX_ENTER(&rx_refcnt_mutex);
-        conn->refCount--;
-        MUTEX_EXIT(&rx_refcnt_mutex);
+       putConnection(conn);
         return np;
     }
 
@@ -3313,9 +3291,7 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
             MUTEX_EXIT(&call->lock);
             if (rx_stats_active)
                 rx_atomic_inc(&rx_stats.spuriousPacketsRead);
-            MUTEX_ENTER(&rx_refcnt_mutex);
-            conn->refCount--;
-            MUTEX_EXIT(&rx_refcnt_mutex);
+           putConnection(conn);
             return np;
         } else if (np->header.callNumber != currentCallNumber) {
            /* Wait until the transmit queue is idle before deciding
@@ -3334,9 +3310,7 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
                 if (call->error) {
                     rxi_CallError(call, call->error);
                     MUTEX_EXIT(&call->lock);
-                    MUTEX_ENTER(&rx_refcnt_mutex);
-                    conn->refCount--;
-                    MUTEX_EXIT(&rx_refcnt_mutex);
+                   putConnection(conn);
                     return np;
                 }
             }
@@ -3351,9 +3325,7 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
                tp = rxi_SendSpecial(call, conn, np, RX_PACKET_TYPE_BUSY,
                                     NULL, 0, 1);
                MUTEX_EXIT(&call->lock);
-                MUTEX_ENTER(&rx_refcnt_mutex);
-               conn->refCount--;
-                MUTEX_EXIT(&rx_refcnt_mutex);
+               putConnection(conn);
                return tp;
            }
            rxi_ResetCall(call, 0);
@@ -3380,9 +3352,7 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
                rxi_CallError(call, rx_BusyError);
                tp = rxi_SendCallAbort(call, np, 1, 0);
                MUTEX_EXIT(&call->lock);
-                MUTEX_ENTER(&rx_refcnt_mutex);
-               conn->refCount--;
-                MUTEX_EXIT(&rx_refcnt_mutex);
+               putConnection(conn);
                 if (rx_stats_active)
                     rx_atomic_inc(&rx_stats.nBusies);
                return tp;
@@ -3398,9 +3368,7 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
             if (rx_stats_active)
                 rx_atomic_inc(&rx_stats.ignorePacketDally);
             MUTEX_EXIT(&call->lock);
-            MUTEX_ENTER(&rx_refcnt_mutex);
-           conn->refCount--;
-            MUTEX_EXIT(&rx_refcnt_mutex);
+           putConnection(conn);
            return np;
        }
 
@@ -3410,18 +3378,14 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
             if (rx_stats_active)
                 rx_atomic_inc(&rx_stats.spuriousPacketsRead);
             MUTEX_EXIT(&call->lock);
-            MUTEX_ENTER(&rx_refcnt_mutex);
-           conn->refCount--;
-            MUTEX_EXIT(&rx_refcnt_mutex);
+           putConnection(conn);
            return np;
        }
        /* If the service security object index stamped in the packet does not
         * match the connection's security index, ignore the packet */
        if (np->header.securityIndex != conn->securityIndex) {
            MUTEX_EXIT(&call->lock);
-            MUTEX_ENTER(&rx_refcnt_mutex);
-           conn->refCount--;
-            MUTEX_EXIT(&rx_refcnt_mutex);
+           putConnection(conn);
            return np;
        }
 
@@ -3442,9 +3406,7 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
 #ifdef RX_ENABLE_LOCKS
                rxi_SetAcksInTransmitQueue(call);
 #else
-                MUTEX_ENTER(&rx_refcnt_mutex);
-               conn->refCount--;
-                MUTEX_EXIT(&rx_refcnt_mutex);
+               putConnection(conn);
                return np;      /* xmitting; drop packet */
 #endif
            } else {
@@ -3472,9 +3434,7 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
                     if (rx_stats_active)
                         rx_atomic_inc(&rx_stats.spuriousPacketsRead);
                    MUTEX_EXIT(&call->lock);
-                    MUTEX_ENTER(&rx_refcnt_mutex);
-                   conn->refCount--;
-                    MUTEX_EXIT(&rx_refcnt_mutex);
+                   putConnection(conn);
                    return np;
                }
            }
@@ -3536,9 +3496,7 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
        dpf(("rxi_ReceivePacket ABORT rx_DataOf = %d\n", errdata));
        rxi_CallError(call, errdata);
        MUTEX_EXIT(&call->lock);
-        MUTEX_ENTER(&rx_refcnt_mutex);
-       conn->refCount--;
-        MUTEX_EXIT(&rx_refcnt_mutex);
+       putConnection(conn);
        return np;              /* xmitting; drop packet */
     }
     case RX_PACKET_TYPE_BUSY: {
@@ -3555,9 +3513,7 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
        MUTEX_EXIT(&call->lock);
        MUTEX_EXIT(&conn->conn_call_lock);
 
-       MUTEX_ENTER(&rx_refcnt_mutex);
-       conn->refCount--;
-       MUTEX_EXIT(&rx_refcnt_mutex);
+       putConnection(conn);
        return np;
     }
 
@@ -3580,9 +3536,7 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
            break;
 #else /* RX_ENABLE_LOCKS */
            MUTEX_EXIT(&call->lock);
-            MUTEX_ENTER(&rx_refcnt_mutex);
-           conn->refCount--;
-            MUTEX_EXIT(&rx_refcnt_mutex);
+           putConnection(conn);
            return np;          /* xmitting; drop packet */
 #endif /* RX_ENABLE_LOCKS */
        }
@@ -3604,9 +3558,7 @@ rxi_ReceivePacket(struct rx_packet *np, osi_socket socket,
     /* we've received a legit packet, so the channel is not busy */
     call->flags &= ~RX_CALL_PEER_BUSY;
     MUTEX_EXIT(&call->lock);
-    MUTEX_ENTER(&rx_refcnt_mutex);
-    conn->refCount--;
-    MUTEX_EXIT(&rx_refcnt_mutex);
+    putConnection(conn);
     return np;
 }
 
@@ -3698,12 +3650,15 @@ rxi_CheckReachEvent(struct rxevent *event, void *arg1, void *arg2, int dummy)
     int i, waiting;
 
     MUTEX_ENTER(&conn->conn_data_lock);
-    conn->checkReachEvent = NULL;
+
+    if (event) {
+       rxevent_Put(conn->checkReachEvent);
+       conn->checkReachEvent = NULL;
+    }
+
     waiting = conn->flags & RX_CONN_ATTACHWAIT;
     if (event) {
-        MUTEX_ENTER(&rx_refcnt_mutex);
-       conn->refCount--;
-        MUTEX_EXIT(&rx_refcnt_mutex);
+       putConnection(conn);
     }
     MUTEX_EXIT(&conn->conn_data_lock);
 
@@ -4845,9 +4800,7 @@ rxi_AttachServerProc(struct rx_call *call,
            *tnop = sq->tno;
            *sq->socketp = socket;
            clock_GetTime(&call->startTime);
-            MUTEX_ENTER(&rx_refcnt_mutex);
            CALL_HOLD(call, RX_CALL_REFCOUNT_BEGIN);
-            MUTEX_EXIT(&rx_refcnt_mutex);
        } else {
            sq->newcall = call;
        }
@@ -4904,10 +4857,9 @@ rxi_AckAll(struct rxevent *event, struct rx_call *call, char *dummy)
 #ifdef RX_ENABLE_LOCKS
     if (event) {
        MUTEX_ENTER(&call->lock);
+       rxevent_Put(call->delayedAckEvent);
        call->delayedAckEvent = NULL;
-        MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_RELE(call, RX_CALL_REFCOUNT_ACKALL);
-        MUTEX_EXIT(&rx_refcnt_mutex);
     }
     rxi_SendSpecial(call, call->conn, (struct rx_packet *)0,
                    RX_PACKET_TYPE_ACKALL, NULL, 0, 0);
@@ -4915,8 +4867,10 @@ rxi_AckAll(struct rxevent *event, struct rx_call *call, char *dummy)
     if (event)
        MUTEX_EXIT(&call->lock);
 #else /* RX_ENABLE_LOCKS */
-    if (event)
+    if (event) {
+       rxevent_Put(call->delayedAckEvent);
        call->delayedAckEvent = NULL;
+    }
     rxi_SendSpecial(call, call->conn, (struct rx_packet *)0,
                    RX_PACKET_TYPE_ACKALL, NULL, 0, 0);
     call->flags |= RX_CALL_ACKALL_SENT;
@@ -4931,18 +4885,20 @@ rxi_SendDelayedAck(struct rxevent *event, void *arg1, void *unused1,
 #ifdef RX_ENABLE_LOCKS
     if (event) {
        MUTEX_ENTER(&call->lock);
-       if (event == call->delayedAckEvent)
+       if (event == call->delayedAckEvent) {
+           rxevent_Put(call->delayedAckEvent);
            call->delayedAckEvent = NULL;
-        MUTEX_ENTER(&rx_refcnt_mutex);
+       }
        CALL_RELE(call, RX_CALL_REFCOUNT_DELAY);
-        MUTEX_EXIT(&rx_refcnt_mutex);
     }
     (void)rxi_SendAck(call, 0, 0, RX_ACK_DELAY, 0);
     if (event)
        MUTEX_EXIT(&call->lock);
 #else /* RX_ENABLE_LOCKS */
-    if (event)
+    if (event) {
+       rxevent_Put(call->delayedAckEvent);
        call->delayedAckEvent = NULL;
+    }
     (void)rxi_SendAck(call, 0, 0, RX_ACK_DELAY, 0);
 #endif /* RX_ENABLE_LOCKS */
 }
@@ -5082,9 +5038,7 @@ rxi_SendCallAbort(struct rx_call *call, struct rx_packet *packet,
        clock_GetTime(&now);
        when = now;
        clock_Addmsec(&when, rxi_callAbortDelay);
-        MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_HOLD(call, RX_CALL_REFCOUNT_ABORT);
-        MUTEX_EXIT(&rx_refcnt_mutex);
        call->delayedAbortEvent =
            rxevent_Post(&when, &now, rxi_SendDelayedCallAbort, call, 0, 0);
     }
@@ -5156,9 +5110,7 @@ rxi_ConnectionError(struct rx_connection *conn,
        if (conn->checkReachEvent) {
            rxevent_Cancel(&conn->checkReachEvent, NULL, 0);
            conn->flags &= ~(RX_CONN_ATTACHWAIT|RX_CONN_NAT_PING);
-            MUTEX_ENTER(&rx_refcnt_mutex);
-           conn->refCount--;
-            MUTEX_EXIT(&rx_refcnt_mutex);
+           putConnection(conn);
        }
        MUTEX_EXIT(&conn->conn_data_lock);
        for (i = 0; i < RX_MAXCALLS; i++) {
@@ -5735,18 +5687,14 @@ rxi_SendList(struct rx_call *call, struct xmitlist *xmit,
     rxevent_Cancel(&call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
 
     MUTEX_EXIT(&call->lock);
-    MUTEX_ENTER(&rx_refcnt_mutex);
     CALL_HOLD(call, RX_CALL_REFCOUNT_SEND);
-    MUTEX_EXIT(&rx_refcnt_mutex);
     if (xmit->len > 1) {
        rxi_SendPacketList(call, conn, xmit->list, xmit->len, istack);
     } else {
        rxi_SendPacket(call, conn, xmit->list[0], istack);
     }
     MUTEX_ENTER(&call->lock);
-    MUTEX_ENTER(&rx_refcnt_mutex);
     CALL_RELE(call, RX_CALL_REFCOUNT_SEND);
-    MUTEX_EXIT(&rx_refcnt_mutex);
 
     /* Tell the RTO calculation engine that we have sent a packet, and
      * if it was the last one */
@@ -5892,9 +5840,8 @@ rxi_Resend(struct rxevent *event, void *arg0, void *arg1, int istack)
      * structure, since there is no longer a per-call retransmission
      * event pending. */
     if (event == call->resendEvent) {
-        MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_RELE(call, RX_CALL_REFCOUNT_RESEND);
-        MUTEX_EXIT(&rx_refcnt_mutex);
+       rxevent_Put(call->resendEvent);
        call->resendEvent = NULL;
     }
 
@@ -6137,13 +6084,9 @@ rxi_Send(struct rx_call *call, struct rx_packet *p,
 
     /* Actually send the packet, filling in more connection-specific fields */
     MUTEX_EXIT(&call->lock);
-    MUTEX_ENTER(&rx_refcnt_mutex);
     CALL_HOLD(call, RX_CALL_REFCOUNT_SEND);
-    MUTEX_EXIT(&rx_refcnt_mutex);
     rxi_SendPacket(call, conn, p, istack);
-    MUTEX_ENTER(&rx_refcnt_mutex);
     CALL_RELE(call, RX_CALL_REFCOUNT_SEND);
-    MUTEX_EXIT(&rx_refcnt_mutex);
     MUTEX_ENTER(&call->lock);
 
     /* Update last send time for this call (for keep-alive
@@ -6369,6 +6312,7 @@ rxi_NatKeepAliveEvent(struct rxevent *event, void *arg1,
     MUTEX_ENTER(&rx_refcnt_mutex);
     /* Only reschedule ourselves if the connection would not be destroyed */
     if (conn->refCount <= 1) {
+       rxevent_Put(conn->natKeepAliveEvent);
        conn->natKeepAliveEvent = NULL;
         MUTEX_EXIT(&rx_refcnt_mutex);
        MUTEX_EXIT(&conn->conn_data_lock);
@@ -6376,6 +6320,7 @@ rxi_NatKeepAliveEvent(struct rxevent *event, void *arg1,
     } else {
        conn->refCount--; /* drop the reference for this */
         MUTEX_EXIT(&rx_refcnt_mutex);
+       rxevent_Put(conn->natKeepAliveEvent);
        conn->natKeepAliveEvent = NULL;
        rxi_ScheduleNatKeepAliveEvent(conn);
        MUTEX_EXIT(&conn->conn_data_lock);
@@ -6438,12 +6383,14 @@ rxi_KeepAliveEvent(struct rxevent *event, void *arg1, void *dummy,
     struct rx_connection *conn;
     afs_uint32 now;
 
-    MUTEX_ENTER(&rx_refcnt_mutex);
     CALL_RELE(call, RX_CALL_REFCOUNT_ALIVE);
-    MUTEX_EXIT(&rx_refcnt_mutex);
     MUTEX_ENTER(&call->lock);
-    if (event == call->keepAliveEvent)
+
+    if (event == call->keepAliveEvent) {
+       rxevent_Put(call->keepAliveEvent);
        call->keepAliveEvent = NULL;
+    }
+
     now = clock_Sec();
 
 #ifdef RX_ENABLE_LOCKS
@@ -6480,13 +6427,13 @@ rxi_GrowMTUEvent(struct rxevent *event, void *arg1, void *dummy, int dummy2)
     struct rx_call *call = arg1;
     struct rx_connection *conn;
 
-    MUTEX_ENTER(&rx_refcnt_mutex);
     CALL_RELE(call, RX_CALL_REFCOUNT_ALIVE);
-    MUTEX_EXIT(&rx_refcnt_mutex);
     MUTEX_ENTER(&call->lock);
 
-    if (event == call->growMTUEvent)
+    if (event == call->growMTUEvent) {
+       rxevent_Put(call->growMTUEvent);
        call->growMTUEvent = NULL;
+    }
 
 #ifdef RX_ENABLE_LOCKS
     if (rxi_CheckCall(call, 0)) {
@@ -6526,9 +6473,7 @@ rxi_ScheduleKeepAliveEvent(struct rx_call *call)
        clock_GetTime(&now);
        when = now;
        when.sec += call->conn->secondsUntilPing;
-        MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_HOLD(call, RX_CALL_REFCOUNT_ALIVE);
-        MUTEX_EXIT(&rx_refcnt_mutex);
        call->keepAliveEvent =
            rxevent_Post(&when, &now, rxi_KeepAliveEvent, call, NULL, 0);
     }
@@ -6551,9 +6496,7 @@ rxi_ScheduleGrowMTUEvent(struct rx_call *call, int secs)
        }
 
        when.sec += secs;
-        MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_HOLD(call, RX_CALL_REFCOUNT_ALIVE);
-        MUTEX_EXIT(&rx_refcnt_mutex);
        call->growMTUEvent =
            rxevent_Post(&when, &now, rxi_GrowMTUEvent, call, NULL, 0);
     }
@@ -6572,6 +6515,20 @@ rxi_KeepAliveOn(struct rx_call *call)
     rxi_ScheduleKeepAliveEvent(call);
 }
 
+/*
+ * Solely in order that callers not need to include rx_call.h
+ */
+void
+rx_KeepAliveOff(struct rx_call *call)
+{
+    rxi_KeepAliveOff(call);
+}
+void
+rx_KeepAliveOn(struct rx_call *call)
+{
+    rxi_KeepAliveOn(call);
+}
+
 void
 rxi_GrowMTUOn(struct rx_call *call)
 {
@@ -6594,6 +6551,7 @@ rxi_SendDelayedConnAbort(struct rxevent *event, void *arg1, void *unused,
     struct rx_packet *packet;
 
     MUTEX_ENTER(&conn->conn_data_lock);
+    rxevent_Put(conn->delayedAbortEvent);
     conn->delayedAbortEvent = NULL;
     error = htonl(conn->error);
     conn->abortCount++;
@@ -6620,6 +6578,7 @@ rxi_SendDelayedCallAbort(struct rxevent *event, void *arg1, void *dummy,
     struct rx_packet *packet;
 
     MUTEX_ENTER(&call->lock);
+    rxevent_Put(call->delayedAbortEvent);
     call->delayedAbortEvent = NULL;
     error = htonl(call->error);
     call->abortCount++;
@@ -6631,9 +6590,7 @@ rxi_SendDelayedCallAbort(struct rxevent *event, void *arg1, void *dummy,
        rxi_FreePacket(packet);
     }
     MUTEX_EXIT(&call->lock);
-    MUTEX_ENTER(&rx_refcnt_mutex);
     CALL_RELE(call, RX_CALL_REFCOUNT_ABORT);
-    MUTEX_EXIT(&rx_refcnt_mutex);
 }
 
 /* This routine is called periodically (every RX_AUTH_REQUEST_TIMEOUT
@@ -6646,7 +6603,11 @@ rxi_ChallengeEvent(struct rxevent *event,
 {
     struct rx_connection *conn = arg0;
 
-    conn->challengeEvent = NULL;
+    if (event) {
+       rxevent_Put(conn->challengeEvent);
+       conn->challengeEvent = NULL;
+    }
+
     if (RXS_CheckAuthentication(conn->securityObject, conn) != 0) {
        struct rx_packet *packet;
        struct clock when, now;
@@ -7074,7 +7035,7 @@ rxi_ReapConnections(struct rxevent *unused, void *unused1, void *unused2,
 
     when = now;
     when.sec += RX_REAP_TIME;  /* Check every RX_REAP_TIME seconds */
-    rxevent_Post(&when, &now, rxi_ReapConnections, 0, NULL, 0);
+    rxevent_Put(rxevent_Post(&when, &now, rxi_ReapConnections, 0, NULL, 0));
 }