rx: Use a red black tree for the event stack
[openafs.git] / src / rx / rx.c
index f6b8b00..1c5622e 100644 (file)
@@ -78,6 +78,7 @@ extern afs_int32 afs_termState;
 #include "rx_trace.h"
 #include "rx_internal.h"
 #include "rx_stats.h"
+#include "rx_event.h"
 
 #include <afs/rxgen_consts.h>
 
@@ -100,6 +101,14 @@ static void rxi_ComputeRoundTripTime(struct rx_packet *, struct rx_ackPacket *,
                                     struct clock *);
 static void rxi_Resend(struct rxevent *event, void *arg0, void *arg1,
                       int istack);
+static void rxi_SendDelayedAck(struct rxevent *event, void *call,
+                               void *dummy, int dummy2);
+static void rxi_SendDelayedCallAbort(struct rxevent *event, void *arg1,
+                                    void *dummy, int dummy2);
+static void rxi_SendDelayedConnAbort(struct rxevent *event, void *arg1,
+                                    void *unused, int unused2);
+static void rxi_ReapConnections(struct rxevent *unused, void *unused1,
+                               void *unused2, int unused3);
 
 #ifdef RX_ENABLE_LOCKS
 static void rxi_SetAcksInTransmitQueue(struct rx_call *call);
@@ -188,7 +197,6 @@ extern afs_kmutex_t des_init_mutex;
 extern afs_kmutex_t des_random_mutex;
 extern afs_kmutex_t rx_clock_mutex;
 extern afs_kmutex_t rxi_connCacheMutex;
-extern afs_kmutex_t rx_event_mutex;
 extern afs_kmutex_t event_handler_mutex;
 extern afs_kmutex_t listener_mutex;
 extern afs_kmutex_t rx_if_init_mutex;
@@ -214,7 +222,6 @@ rxi_InitPthread(void)
     MUTEX_INIT(&rx_refcnt_mutex, "refcnts", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&epoch_mutex, "epoch", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rx_init_mutex, "init", MUTEX_DEFAULT, 0);
-    MUTEX_INIT(&rx_event_mutex, "event", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&event_handler_mutex, "event handler", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&rxi_connCacheMutex, "conn cache", MUTEX_DEFAULT, 0);
     MUTEX_INIT(&listener_mutex, "listener", MUTEX_DEFAULT, 0);
@@ -648,8 +655,8 @@ rxi_rto_startTimer(struct rx_call *call, int lastPacket, int istack)
     MUTEX_ENTER(&rx_refcnt_mutex);
     CALL_HOLD(call, RX_CALL_REFCOUNT_RESEND);
     MUTEX_EXIT(&rx_refcnt_mutex);
-    call->resendEvent = rxevent_PostNow2(&retryTime, &now, rxi_Resend,
-                                        call, 0, istack);
+    call->resendEvent = rxevent_Post(&retryTime, &now, rxi_Resend,
+                                    call, NULL, istack);
 }
 
 /*!
@@ -666,10 +673,7 @@ rxi_rto_startTimer(struct rx_call *call, int lastPacket, int istack)
 static_inline void
 rxi_rto_cancel(struct rx_call *call)
 {
-    if (!call->resendEvent)
-       return;
-
-    rxevent_Cancel(call->resendEvent, call, RX_CALL_REFCOUNT_RESEND);
+    rxevent_Cancel(&call->resendEvent, call, RX_CALL_REFCOUNT_RESEND);
 }
 
 /*!
@@ -755,6 +759,37 @@ rx_SetBusyChannelError(afs_int32 error)
     rxi_busyChannelError = error;
 }
 
+/**
+ * Set a delayed ack event on the specified call for the given time
+ *
+ * @param[in] call - the call on which to set the event
+ * @param[in] offset - the delay from now after which the event fires
+ */
+void
+rxi_PostDelayedAckEvent(struct rx_call *call, struct clock *offset)
+{
+    struct clock now, when;
+
+    clock_GetTime(&now);
+    when = now;
+    clock_Add(&when, offset);
+
+    if (!call->delayedAckEvent
+       || clock_Gt(&call->delayedAckTime, &when)) {
+
+        rxevent_Cancel(&call->delayedAckEvent, call,
+                      RX_CALL_REFCOUNT_DELAY);
+       MUTEX_ENTER(&rx_refcnt_mutex);
+       CALL_HOLD(call, RX_CALL_REFCOUNT_DELAY);
+       MUTEX_EXIT(&rx_refcnt_mutex);
+
+       call->delayedAckEvent = rxevent_Post(&when, &now,
+                                            rxi_SendDelayedAck,
+                                            call, NULL, 0);
+       call->delayedAckTime = when;
+    }
+}
+
 /* called with unincremented nRequestsRunning to see if it is OK to start
  * a new thread in this service.  Could be "no" for two reasons: over the
  * max quota, or would prevent others from reaching their min quota.
@@ -914,7 +949,7 @@ rx_StartServer(int donateMe)
     }
 
     /* Turn on reaping of idle server connections */
-    rxi_ReapConnections(NULL, NULL, NULL);
+    rxi_ReapConnections(NULL, NULL, NULL, 0);
 
     USERPRI;
 
@@ -1213,7 +1248,7 @@ rxi_DestroyConnectionNoLock(struct rx_connection *conn)
                    /* Push the final acknowledgment out now--there
                     * won't be a subsequent call to acknowledge the
                     * last reply packets */
-                   rxevent_Cancel(call->delayedAckEvent, call,
+                   rxevent_Cancel(&call->delayedAckEvent, call,
                                   RX_CALL_REFCOUNT_DELAY);
                    if (call->state == RX_STATE_PRECALL
                        || call->state == RX_STATE_ACTIVE) {
@@ -1254,7 +1289,7 @@ rxi_DestroyConnectionNoLock(struct rx_connection *conn)
     }
 
     if (conn->delayedAbortEvent) {
-       rxevent_Cancel(conn->delayedAbortEvent, (struct rx_call *)0, 0);
+       rxevent_Cancel(&conn->delayedAbortEvent, NULL, 0);
        packet = rxi_AllocPacket(RX_PACKET_CLASS_SPECIAL);
        if (packet) {
            MUTEX_ENTER(&conn->conn_data_lock);
@@ -1282,12 +1317,9 @@ rxi_DestroyConnectionNoLock(struct rx_connection *conn)
 
     /* Make sure the connection is completely reset before deleting it. */
     /* get rid of pending events that could zap us later */
-    if (conn->challengeEvent)
-       rxevent_Cancel(conn->challengeEvent, (struct rx_call *)0, 0);
-    if (conn->checkReachEvent)
-       rxevent_Cancel(conn->checkReachEvent, (struct rx_call *)0, 0);
-    if (conn->natKeepAliveEvent)
-       rxevent_Cancel(conn->natKeepAliveEvent, (struct rx_call *)0, 0);
+    rxevent_Cancel(&conn->challengeEvent, NULL, 0);
+    rxevent_Cancel(&conn->checkReachEvent, NULL, 0);
+    rxevent_Cancel(&conn->natKeepAliveEvent, NULL, 0);
 
     /* Add the connection to the list of destroyed connections that
      * need to be cleaned up. This is necessary to avoid deadlocks
@@ -2335,7 +2367,7 @@ rx_EndCall(struct rx_call *call, afs_int32 rc)
            call->state = RX_STATE_DALLY;
            rxi_ClearTransmitQueue(call, 0);
            rxi_rto_cancel(call);
-           rxevent_Cancel(call->keepAliveEvent, call,
+           rxevent_Cancel(&call->keepAliveEvent, call,
                           RX_CALL_REFCOUNT_ALIVE);
        }
     } else {                   /* Client connection */
@@ -2353,10 +2385,9 @@ rx_EndCall(struct rx_call *call, afs_int32 rc)
         * and force-send it now.
         */
        if (call->delayedAckEvent) {
-           rxevent_Cancel(call->delayedAckEvent, call,
+           rxevent_Cancel(&call->delayedAckEvent, call,
                           RX_CALL_REFCOUNT_DELAY);
-           call->delayedAckEvent = NULL;
-           rxi_SendDelayedAck(NULL, call, NULL);
+           rxi_SendDelayedAck(NULL, call, NULL, 0);
        }
 
        /* We need to release the call lock since it's lower than the
@@ -3656,7 +3687,7 @@ rxi_ConnClearAttachWait(struct rx_connection *conn)
 }
 
 static void
-rxi_CheckReachEvent(struct rxevent *event, void *arg1, void *arg2)
+rxi_CheckReachEvent(struct rxevent *event, void *arg1, void *arg2, int dummy)
 {
     struct rx_connection *conn = arg1;
     struct rx_call *acall = arg2;
@@ -3665,7 +3696,12 @@ rxi_CheckReachEvent(struct rxevent *event, void *arg1, void *arg2)
     int i, waiting;
 
     MUTEX_ENTER(&conn->conn_data_lock);
-    conn->checkReachEvent = NULL;
+
+    if (event) {
+       rxevent_Put(conn->checkReachEvent);
+       conn->checkReachEvent = NULL;
+    }
+
     waiting = conn->flags & RX_CONN_ATTACHWAIT;
     if (event) {
         MUTEX_ENTER(&rx_refcnt_mutex);
@@ -3706,9 +3742,9 @@ rxi_CheckReachEvent(struct rxevent *event, void *arg1, void *arg2)
                 MUTEX_ENTER(&rx_refcnt_mutex);
                conn->refCount++;
                 MUTEX_EXIT(&rx_refcnt_mutex);
-               conn->checkReachEvent =
-                   rxevent_PostNow(&when, &now, rxi_CheckReachEvent, conn,
-                                   NULL);
+               conn->checkReachEvent = rxevent_Post(&when, &now,
+                                                    rxi_CheckReachEvent, conn,
+                                                    NULL, 0);
            }
            MUTEX_EXIT(&conn->conn_data_lock);
        }
@@ -3740,7 +3776,7 @@ rxi_CheckConnReach(struct rx_connection *conn, struct rx_call *call)
     conn->flags |= RX_CONN_ATTACHWAIT;
     MUTEX_EXIT(&conn->conn_data_lock);
     if (!conn->checkReachEvent)
-       rxi_CheckReachEvent(NULL, conn, call);
+       rxi_CheckReachEvent(NULL, conn, call, 0);
 
     return 1;
 }
@@ -3786,7 +3822,6 @@ rxi_ReceiveDataPacket(struct rx_call *call,
     afs_uint32 serial=0, flags=0;
     int isFirst;
     struct rx_packet *tnp;
-    struct clock when, now;
     if (rx_stats_active)
         rx_atomic_inc(&rx_stats.dataPacketsRead);
 
@@ -3806,20 +3841,8 @@ rxi_ReceiveDataPacket(struct rx_call *call,
         /* We used to clear the receive queue here, in an attempt to free
          * packets. However this is unsafe if the queue has received a
          * soft ACK for the final packet */
-       clock_GetTime(&now);
-       when = now;
-       clock_Add(&when, &rx_softAckDelay);
-       if (!call->delayedAckEvent
-           || clock_Gt(&call->delayedAckEvent->eventTime, &when)) {
-           rxevent_Cancel(call->delayedAckEvent, call,
-                          RX_CALL_REFCOUNT_DELAY);
-            MUTEX_ENTER(&rx_refcnt_mutex);
-           CALL_HOLD(call, RX_CALL_REFCOUNT_DELAY);
-            MUTEX_EXIT(&rx_refcnt_mutex);
+       rxi_PostDelayedAckEvent(call, &rx_softAckDelay);
 
-           call->delayedAckEvent =
-               rxevent_PostNow(&when, &now, rxi_SendDelayedAck, call, 0);
-       }
        /* we've damaged this call already, might as well do it in. */
        return np;
     }
@@ -3871,7 +3894,7 @@ rxi_ReceiveDataPacket(struct rx_call *call,
                 if (rx_stats_active)
                     rx_atomic_inc(&rx_stats.dupPacketsRead);
                dpf(("packet %"AFS_PTR_FMT" dropped on receipt - duplicate\n", np));
-               rxevent_Cancel(call->delayedAckEvent, call,
+               rxevent_Cancel(&call->delayedAckEvent, call,
                               RX_CALL_REFCOUNT_DELAY);
                np = rxi_SendAck(call, np, serial, RX_ACK_DUPLICATE, istack);
                ackNeeded = 0;
@@ -3961,7 +3984,7 @@ rxi_ReceiveDataPacket(struct rx_call *call,
            if (seq < call->rnext) {
                 if (rx_stats_active)
                     rx_atomic_inc(&rx_stats.dupPacketsRead);
-               rxevent_Cancel(call->delayedAckEvent, call,
+               rxevent_Cancel(&call->delayedAckEvent, call,
                               RX_CALL_REFCOUNT_DELAY);
                np = rxi_SendAck(call, np, serial, RX_ACK_DUPLICATE, istack);
                ackNeeded = 0;
@@ -3973,7 +3996,7 @@ rxi_ReceiveDataPacket(struct rx_call *call,
             * accomodated by the current window, then send a negative
             * acknowledge and drop the packet */
            if ((call->rnext + call->rwind) <= seq) {
-               rxevent_Cancel(call->delayedAckEvent, call,
+               rxevent_Cancel(&call->delayedAckEvent, call,
                               RX_CALL_REFCOUNT_DELAY);
                np = rxi_SendAck(call, np, serial, RX_ACK_EXCEEDS_WINDOW,
                                 istack);
@@ -3989,7 +4012,7 @@ rxi_ReceiveDataPacket(struct rx_call *call,
                if (seq == tp->header.seq) {
                     if (rx_stats_active)
                         rx_atomic_inc(&rx_stats.dupPacketsRead);
-                   rxevent_Cancel(call->delayedAckEvent, call,
+                   rxevent_Cancel(&call->delayedAckEvent, call,
                                   RX_CALL_REFCOUNT_DELAY);
                    np = rxi_SendAck(call, np, serial, RX_ACK_DUPLICATE,
                                     istack);
@@ -4099,31 +4122,18 @@ rxi_ReceiveDataPacket(struct rx_call *call,
      * received. Always send a soft ack for the last packet in
      * the server's reply. */
     if (ackNeeded) {
-       rxevent_Cancel(call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
+       rxevent_Cancel(&call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
        np = rxi_SendAck(call, np, serial, ackNeeded, istack);
     } else if (call->nSoftAcks > (u_short) rxi_SoftAckRate) {
-       rxevent_Cancel(call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
+       rxevent_Cancel(&call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
        np = rxi_SendAck(call, np, serial, RX_ACK_IDLE, istack);
     } else if (call->nSoftAcks) {
-       clock_GetTime(&now);
-       when = now;
-       if (haveLast && !(flags & RX_CLIENT_INITIATED)) {
-           clock_Add(&when, &rx_lastAckDelay);
-       } else {
-           clock_Add(&when, &rx_softAckDelay);
-       }
-       if (!call->delayedAckEvent
-           || clock_Gt(&call->delayedAckEvent->eventTime, &when)) {
-           rxevent_Cancel(call->delayedAckEvent, call,
-                          RX_CALL_REFCOUNT_DELAY);
-            MUTEX_ENTER(&rx_refcnt_mutex);
-           CALL_HOLD(call, RX_CALL_REFCOUNT_DELAY);
-            MUTEX_EXIT(&rx_refcnt_mutex);
-           call->delayedAckEvent =
-               rxevent_PostNow(&when, &now, rxi_SendDelayedAck, call, 0);
-       }
+       if (haveLast && !(flags & RX_CLIENT_INITIATED))
+           rxi_PostDelayedAckEvent(call, &rx_lastAckDelay);
+       else
+           rxi_PostDelayedAckEvent(call, &rx_softAckDelay);
     } else if (call->flags & RX_CALL_RECEIVE_DONE) {
-       rxevent_Cancel(call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
+       rxevent_Cancel(&call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
     }
 
     return np;
@@ -4688,7 +4698,7 @@ rxi_ReceiveAckPacket(struct rx_call *call, struct rx_packet *np,
        && call->tfirst + call->nSoftAcked >= call->tnext) {
        call->state = RX_STATE_DALLY;
        rxi_ClearTransmitQueue(call, 0);
-        rxevent_Cancel(call->keepAliveEvent, call, RX_CALL_REFCOUNT_ALIVE);
+        rxevent_Cancel(&call->keepAliveEvent, call, RX_CALL_REFCOUNT_ALIVE);
     } else if (!queue_IsEmpty(&call->tq)) {
        rxi_Start(call, istack);
     }
@@ -4897,6 +4907,7 @@ rxi_AckAll(struct rxevent *event, struct rx_call *call, char *dummy)
 #ifdef RX_ENABLE_LOCKS
     if (event) {
        MUTEX_ENTER(&call->lock);
+       rxevent_Put(call->delayedAckEvent);
        call->delayedAckEvent = NULL;
         MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_RELE(call, RX_CALL_REFCOUNT_ACKALL);
@@ -4908,8 +4919,10 @@ rxi_AckAll(struct rxevent *event, struct rx_call *call, char *dummy)
     if (event)
        MUTEX_EXIT(&call->lock);
 #else /* RX_ENABLE_LOCKS */
-    if (event)
+    if (event) {
+       rxevent_Put(call->delayedAckEvent);
        call->delayedAckEvent = NULL;
+    }
     rxi_SendSpecial(call, call->conn, (struct rx_packet *)0,
                    RX_PACKET_TYPE_ACKALL, NULL, 0, 0);
     call->flags |= RX_CALL_ACKALL_SENT;
@@ -4917,14 +4930,17 @@ rxi_AckAll(struct rxevent *event, struct rx_call *call, char *dummy)
 }
 
 void
-rxi_SendDelayedAck(struct rxevent *event, void *arg1, void *unused)
+rxi_SendDelayedAck(struct rxevent *event, void *arg1, void *unused1,
+                  int unused2)
 {
     struct rx_call *call = arg1;
 #ifdef RX_ENABLE_LOCKS
     if (event) {
        MUTEX_ENTER(&call->lock);
-       if (event == call->delayedAckEvent)
+       if (event == call->delayedAckEvent) {
+           rxevent_Put(call->delayedAckEvent);
            call->delayedAckEvent = NULL;
+       }
         MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_RELE(call, RX_CALL_REFCOUNT_DELAY);
         MUTEX_EXIT(&rx_refcnt_mutex);
@@ -4933,8 +4949,10 @@ rxi_SendDelayedAck(struct rxevent *event, void *arg1, void *unused)
     if (event)
        MUTEX_EXIT(&call->lock);
 #else /* RX_ENABLE_LOCKS */
-    if (event)
+    if (event) {
+       rxevent_Put(call->delayedAckEvent);
        call->delayedAckEvent = NULL;
+    }
     (void)rxi_SendAck(call, 0, 0, RX_ACK_DELAY, 0);
 #endif /* RX_ENABLE_LOCKS */
 }
@@ -5062,7 +5080,7 @@ rxi_SendCallAbort(struct rx_call *call, struct rx_packet *packet,
     if (force || rxi_callAbortThreshhold == 0
        || call->abortCount < rxi_callAbortThreshhold) {
        if (call->delayedAbortEvent) {
-           rxevent_Cancel(call->delayedAbortEvent, call,
+           rxevent_Cancel(&call->delayedAbortEvent, call,
                           RX_CALL_REFCOUNT_ABORT);
        }
        error = htonl(call->error);
@@ -5078,7 +5096,7 @@ rxi_SendCallAbort(struct rx_call *call, struct rx_packet *packet,
        CALL_HOLD(call, RX_CALL_REFCOUNT_ABORT);
         MUTEX_EXIT(&rx_refcnt_mutex);
        call->delayedAbortEvent =
-           rxevent_PostNow(&when, &now, rxi_SendDelayedCallAbort, call, 0);
+           rxevent_Post(&when, &now, rxi_SendDelayedCallAbort, call, 0, 0);
     }
     return packet;
 }
@@ -5108,9 +5126,8 @@ rxi_SendConnectionAbort(struct rx_connection *conn,
 
     if (force || rxi_connAbortThreshhold == 0
        || conn->abortCount < rxi_connAbortThreshhold) {
-       if (conn->delayedAbortEvent) {
-           rxevent_Cancel(conn->delayedAbortEvent, (struct rx_call *)0, 0);
-       }
+
+       rxevent_Cancel(&conn->delayedAbortEvent, NULL, 0);
        error = htonl(conn->error);
        conn->abortCount++;
        MUTEX_EXIT(&conn->conn_data_lock);
@@ -5124,7 +5141,7 @@ rxi_SendConnectionAbort(struct rx_connection *conn,
        when = now;
        clock_Addmsec(&when, rxi_connAbortDelay);
        conn->delayedAbortEvent =
-           rxevent_PostNow(&when, &now, rxi_SendDelayedConnAbort, conn, 0);
+           rxevent_Post(&when, &now, rxi_SendDelayedConnAbort, conn, NULL, 0);
     }
     return packet;
 }
@@ -5144,13 +5161,10 @@ rxi_ConnectionError(struct rx_connection *conn,
        dpf(("rxi_ConnectionError conn %"AFS_PTR_FMT" error %d\n", conn, error));
 
        MUTEX_ENTER(&conn->conn_data_lock);
-       if (conn->challengeEvent)
-           rxevent_Cancel(conn->challengeEvent, (struct rx_call *)0, 0);
-       if (conn->natKeepAliveEvent)
-           rxevent_Cancel(conn->natKeepAliveEvent, (struct rx_call *)0, 0);
+       rxevent_Cancel(&conn->challengeEvent, NULL, 0);
+       rxevent_Cancel(&conn->natKeepAliveEvent, NULL, 0);
        if (conn->checkReachEvent) {
-           rxevent_Cancel(conn->checkReachEvent, (struct rx_call *)0, 0);
-           conn->checkReachEvent = 0;
+           rxevent_Cancel(&conn->checkReachEvent, NULL, 0);
            conn->flags &= ~(RX_CONN_ATTACHWAIT|RX_CONN_NAT_PING);
             MUTEX_ENTER(&rx_refcnt_mutex);
            conn->refCount--;
@@ -5231,12 +5245,11 @@ rxi_ResetCall(struct rx_call *call, int newcall)
        call->arrivalProc = (void (*)())0;
     }
 
-    if (call->growMTUEvent)
-       rxevent_Cancel(call->growMTUEvent, call,
-                      RX_CALL_REFCOUNT_ALIVE);
+
+    rxevent_Cancel(&call->growMTUEvent, call, RX_CALL_REFCOUNT_ALIVE);
 
     if (call->delayedAbortEvent) {
-       rxevent_Cancel(call->delayedAbortEvent, call, RX_CALL_REFCOUNT_ABORT);
+       rxevent_Cancel(&call->delayedAbortEvent, call, RX_CALL_REFCOUNT_ABORT);
        packet = rxi_AllocPacket(RX_PACKET_CLASS_SPECIAL);
        if (packet) {
            rxi_SendCallAbort(call, packet, 0, 1);
@@ -5366,7 +5379,7 @@ rxi_ResetCall(struct rx_call *call, int newcall)
 #endif /* RX_ENABLE_LOCKS */
 
     rxi_KeepAliveOff(call);
-    rxevent_Cancel(call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
+    rxevent_Cancel(&call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
 }
 
 /* Send an acknowledge for the indicated packet (seq,serial) of the
@@ -5729,7 +5742,7 @@ rxi_SendList(struct rx_call *call, struct xmitlist *xmit,
 
     /* Since we're about to send a data packet to the peer, it's
      * safe to nuke any scheduled end-of-packets ack */
-    rxevent_Cancel(call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
+    rxevent_Cancel(&call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
 
     MUTEX_EXIT(&call->lock);
     MUTEX_ENTER(&rx_refcnt_mutex);
@@ -5892,6 +5905,7 @@ rxi_Resend(struct rxevent *event, void *arg0, void *arg1, int istack)
         MUTEX_ENTER(&rx_refcnt_mutex);
        CALL_RELE(call, RX_CALL_REFCOUNT_RESEND);
         MUTEX_EXIT(&rx_refcnt_mutex);
+       rxevent_Put(call->resendEvent);
        call->resendEvent = NULL;
     }
 
@@ -6130,7 +6144,7 @@ rxi_Send(struct rx_call *call, struct rx_packet *p,
 
     /* Since we're about to send SOME sort of packet to the peer, it's
      * safe to nuke any scheduled end-of-packets ack */
-    rxevent_Cancel(call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
+    rxevent_Cancel(&call->delayedAckEvent, call, RX_CALL_REFCOUNT_DELAY);
 
     /* Actually send the packet, filling in more connection-specific fields */
     MUTEX_EXIT(&call->lock);
@@ -6233,14 +6247,13 @@ rxi_CheckCall(struct rx_call *call)
        } else {
 #ifdef RX_ENABLE_LOCKS
            /* Cancel pending events */
-           rxevent_Cancel(call->delayedAckEvent, call,
+           rxevent_Cancel(&call->delayedAckEvent, call,
                           RX_CALL_REFCOUNT_DELAY);
            rxi_rto_cancel(call);
-           rxevent_Cancel(call->keepAliveEvent, call,
+           rxevent_Cancel(&call->keepAliveEvent, call,
+                          RX_CALL_REFCOUNT_ALIVE);
+           rxevent_Cancel(&call->growMTUEvent, call,
                           RX_CALL_REFCOUNT_ALIVE);
-           if (call->growMTUEvent)
-               rxevent_Cancel(call->growMTUEvent, call,
-                              RX_CALL_REFCOUNT_ALIVE);
             MUTEX_ENTER(&rx_refcnt_mutex);
            if (call->refCount == 0) {
                rxi_FreeCall(call, haveCTLock);
@@ -6324,7 +6337,8 @@ mtuout:
 }
 
 void
-rxi_NatKeepAliveEvent(struct rxevent *event, void *arg1, void *dummy)
+rxi_NatKeepAliveEvent(struct rxevent *event, void *arg1,
+                     void *dummy, int dummy2)
 {
     struct rx_connection *conn = arg1;
     struct rx_header theader;
@@ -6366,6 +6380,7 @@ rxi_NatKeepAliveEvent(struct rxevent *event, void *arg1, void *dummy)
     MUTEX_ENTER(&rx_refcnt_mutex);
     /* Only reschedule ourselves if the connection would not be destroyed */
     if (conn->refCount <= 1) {
+       rxevent_Put(conn->natKeepAliveEvent);
        conn->natKeepAliveEvent = NULL;
         MUTEX_EXIT(&rx_refcnt_mutex);
        MUTEX_EXIT(&conn->conn_data_lock);
@@ -6373,6 +6388,7 @@ rxi_NatKeepAliveEvent(struct rxevent *event, void *arg1, void *dummy)
     } else {
        conn->refCount--; /* drop the reference for this */
         MUTEX_EXIT(&rx_refcnt_mutex);
+       rxevent_Put(conn->natKeepAliveEvent);
        conn->natKeepAliveEvent = NULL;
        rxi_ScheduleNatKeepAliveEvent(conn);
        MUTEX_EXIT(&conn->conn_data_lock);
@@ -6391,7 +6407,7 @@ rxi_ScheduleNatKeepAliveEvent(struct rx_connection *conn)
        conn->refCount++; /* hold a reference for this */
         MUTEX_EXIT(&rx_refcnt_mutex);
        conn->natKeepAliveEvent =
-           rxevent_PostNow(&when, &now, rxi_NatKeepAliveEvent, conn, 0);
+           rxevent_Post(&when, &now, rxi_NatKeepAliveEvent, conn, NULL, 0);
     }
 }
 
@@ -6428,7 +6444,8 @@ rxi_NatKeepAliveOn(struct rx_connection *conn)
  * keep-alive packet (if we're actually trying to keep the call alive)
  */
 void
-rxi_KeepAliveEvent(struct rxevent *event, void *arg1, void *dummy)
+rxi_KeepAliveEvent(struct rxevent *event, void *arg1, void *dummy,
+                  int dummy2)
 {
     struct rx_call *call = arg1;
     struct rx_connection *conn;
@@ -6438,8 +6455,12 @@ rxi_KeepAliveEvent(struct rxevent *event, void *arg1, void *dummy)
     CALL_RELE(call, RX_CALL_REFCOUNT_ALIVE);
     MUTEX_EXIT(&rx_refcnt_mutex);
     MUTEX_ENTER(&call->lock);
-    if (event == call->keepAliveEvent)
+
+    if (event == call->keepAliveEvent) {
+       rxevent_Put(call->keepAliveEvent);
        call->keepAliveEvent = NULL;
+    }
+
     now = clock_Sec();
 
 #ifdef RX_ENABLE_LOCKS
@@ -6471,7 +6492,7 @@ rxi_KeepAliveEvent(struct rxevent *event, void *arg1, void *dummy)
 
 /* Does what's on the nameplate. */
 void
-rxi_GrowMTUEvent(struct rxevent *event, void *arg1, void *dummy)
+rxi_GrowMTUEvent(struct rxevent *event, void *arg1, void *dummy, int dummy2)
 {
     struct rx_call *call = arg1;
     struct rx_connection *conn;
@@ -6481,8 +6502,10 @@ rxi_GrowMTUEvent(struct rxevent *event, void *arg1, void *dummy)
     MUTEX_EXIT(&rx_refcnt_mutex);
     MUTEX_ENTER(&call->lock);
 
-    if (event == call->growMTUEvent)
+    if (event == call->growMTUEvent) {
+       rxevent_Put(call->growMTUEvent);
        call->growMTUEvent = NULL;
+    }
 
 #ifdef RX_ENABLE_LOCKS
     if (rxi_CheckCall(call, 0)) {
@@ -6526,7 +6549,7 @@ rxi_ScheduleKeepAliveEvent(struct rx_call *call)
        CALL_HOLD(call, RX_CALL_REFCOUNT_ALIVE);
         MUTEX_EXIT(&rx_refcnt_mutex);
        call->keepAliveEvent =
-           rxevent_PostNow(&when, &now, rxi_KeepAliveEvent, call, 0);
+           rxevent_Post(&when, &now, rxi_KeepAliveEvent, call, NULL, 0);
     }
 }
 
@@ -6551,7 +6574,7 @@ rxi_ScheduleGrowMTUEvent(struct rx_call *call, int secs)
        CALL_HOLD(call, RX_CALL_REFCOUNT_ALIVE);
         MUTEX_EXIT(&rx_refcnt_mutex);
        call->growMTUEvent =
-           rxevent_PostNow(&when, &now, rxi_GrowMTUEvent, call, 0);
+           rxevent_Post(&when, &now, rxi_GrowMTUEvent, call, NULL, 0);
     }
 }
 
@@ -6581,8 +6604,8 @@ rxi_GrowMTUOn(struct rx_call *call)
 /* This routine is called to send connection abort messages
  * that have been delayed to throttle looping clients. */
 void
-rxi_SendDelayedConnAbort(struct rxevent *event,
-                        void *arg1, void *unused)
+rxi_SendDelayedConnAbort(struct rxevent *event, void *arg1, void *unused,
+                        int unused2)
 {
     struct rx_connection *conn = arg1;
 
@@ -6590,6 +6613,7 @@ rxi_SendDelayedConnAbort(struct rxevent *event,
     struct rx_packet *packet;
 
     MUTEX_ENTER(&conn->conn_data_lock);
+    rxevent_Put(conn->delayedAbortEvent);
     conn->delayedAbortEvent = NULL;
     error = htonl(conn->error);
     conn->abortCount++;
@@ -6606,9 +6630,9 @@ rxi_SendDelayedConnAbort(struct rxevent *event,
 
 /* This routine is called to send call abort messages
  * that have been delayed to throttle looping clients. */
-void
-rxi_SendDelayedCallAbort(struct rxevent *event,
-                        void *arg1, void *dummy)
+static void
+rxi_SendDelayedCallAbort(struct rxevent *event, void *arg1, void *dummy,
+                        int dummy2)
 {
     struct rx_call *call = arg1;
 
@@ -6616,6 +6640,7 @@ rxi_SendDelayedCallAbort(struct rxevent *event,
     struct rx_packet *packet;
 
     MUTEX_ENTER(&call->lock);
+    rxevent_Put(call->delayedAbortEvent);
     call->delayedAbortEvent = NULL;
     error = htonl(call->error);
     call->abortCount++;
@@ -6642,7 +6667,11 @@ rxi_ChallengeEvent(struct rxevent *event,
 {
     struct rx_connection *conn = arg0;
 
-    conn->challengeEvent = NULL;
+    if (event) {
+       rxevent_Put(conn->challengeEvent);
+       conn->challengeEvent = NULL;
+    }
+
     if (RXS_CheckAuthentication(conn->securityObject, conn) != 0) {
        struct rx_packet *packet;
        struct clock when, now;
@@ -6682,7 +6711,7 @@ rxi_ChallengeEvent(struct rxevent *event,
        when = now;
        when.sec += RX_CHALLENGE_TIMEOUT;
        conn->challengeEvent =
-           rxevent_PostNow2(&when, &now, rxi_ChallengeEvent, conn, 0,
+           rxevent_Post(&when, &now, rxi_ChallengeEvent, conn, 0,
                         (tries - 1));
     }
 }
@@ -6862,7 +6891,8 @@ rxi_ComputeRoundTripTime(struct rx_packet *p,
 /* Find all server connections that have not been active for a long time, and
  * toss them */
 void
-rxi_ReapConnections(struct rxevent *unused, void *unused1, void *unused2)
+rxi_ReapConnections(struct rxevent *unused, void *unused1, void *unused2,
+                   int unused3)
 {
     struct clock now, when;
     clock_GetTime(&now);
@@ -7069,7 +7099,7 @@ rxi_ReapConnections(struct rxevent *unused, void *unused1, void *unused2)
 
     when = now;
     when.sec += RX_REAP_TIME;  /* Check every RX_REAP_TIME seconds */
-    rxevent_Post(&when, rxi_ReapConnections, 0, 0);
+    rxevent_Put(rxevent_Post(&when, &now, rxi_ReapConnections, 0, NULL, 0));
 }