2 * Copyright 2000, International Business Machines Corporation and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
11 /* kdump for SGI needs MP and SP versions of rx_serverQueueEntry,
12 * rx_peer, rx_connection and rx_call structs. rx.h gets included a
13 * second time to pick up mp_ versions of those structs. Currently
14 * the affected struct's have #ifdef's in them for the second pass.
15 * This should change once we start using only ANSI compilers.
16 * Actually, kdump does not use rx_serverQueueEntry, but I'm including
17 * it for completeness.
26 /* Substitute VOID (char) for void, because some compilers are confused by void
27 * in some situations */
33 #include "rx_kmutex.h"
34 #include "rx_kernel.h"
38 #include "rx_packet.h"
41 #if defined (AFS_OBSD_ENV) && !defined (MLEN)
44 #include "netinet/in.h"
45 #include "sys/socket.h"
47 # include <sys/types.h>
49 #ifdef AFS_PTHREAD_ENV
50 # include "rx_pthread.h"
58 # include "rx_clock.h"
59 # include "rx_event.h"
60 # include "rx_packet.h"
63 # include "rx_multi.h"
65 # include <netinet/in.h>
66 # include <sys/socket.h>
71 /* Configurable parameters */
72 #define RX_IDLE_DEAD_TIME 60 /* default idle dead time */
73 #define RX_MAX_SERVICES 20 /* Maximum number of services that may be installed */
74 #if defined(KERNEL) && defined(AFS_AIX51_ENV) && defined(__64__)
75 #define RX_DEFAULT_STACK_SIZE 24000
77 #define RX_DEFAULT_STACK_SIZE 16000 /* Default process stack size; overriden by rx_SetStackSize */
80 /* This parameter should not normally be changed */
81 #define RX_PROCESS_PRIORITY LWP_NORMAL_PRIORITY
83 /* backoff is fixed point binary. Ie, units of 1/4 seconds */
84 #define MAXBACKOFF 0x1F
86 #define ADDRSPERSITE 16
89 /* Bottom n-bits of the Call Identifier give the call number */
90 #define RX_MAXCALLS 4 /* Power of 2; max async calls per connection */
91 #define RX_CIDSHIFT 2 /* Log2(RX_MAXCALLS) */
92 #define RX_CHANNELMASK (RX_MAXCALLS-1)
93 #define RX_CIDMASK (~RX_CHANNELMASK)
94 #endif /* !KDUMP_RX_LOCK */
97 typedef void (*rx_destructor_t) (void *);
98 int rx_KeyCreate(rx_destructor_t);
99 osi_socket rxi_GetHostUDPSocket(u_int host, u_short port);
100 osi_socket rxi_GetUDPSocket(u_short port);
104 int ntoh_syserr_conv(int error);
107 #define RX_DONTWAIT 0
109 #define rx_ConnectionOf(call) ((call)->conn)
110 #define rx_PeerOf(conn) ((conn)->peer)
111 #define rx_HostOf(peer) ((peer)->host)
112 #define rx_PortOf(peer) ((peer)->port)
113 #define rx_SetLocalStatus(call, status) ((call)->localStatus = (status))
114 #define rx_GetLocalStatus(call, status) ((call)->localStatus)
115 #define rx_GetRemoteStatus(call) ((call)->remoteStatus)
116 #define rx_Error(call) ((call)->error)
117 #define rx_ConnError(conn) ((conn)->error)
118 #define rx_IsServerConn(conn) ((conn)->type == RX_SERVER_CONNECTION)
119 #define rx_IsClientConn(conn) ((conn)->type == RX_CLIENT_CONNECTION)
120 /* Don't use these; use the IsServerConn style */
121 #define rx_ServerConn(conn) ((conn)->type == RX_SERVER_CONNECTION)
122 #define rx_ClientConn(conn) ((conn)->type == RX_CLIENT_CONNECTION)
123 #define rx_IsUsingPktCksum(conn) ((conn)->flags & RX_CONN_USING_PACKET_CKSUM)
124 #define rx_ServiceIdOf(conn) ((conn)->serviceId)
125 #define rx_SecurityClassOf(conn) ((conn)->securityIndex)
126 #define rx_SecurityObjectOf(conn) ((conn)->securityObject)
129 * Macros callable by the user to further define attributes of a
130 * service. Must be called before rx_StartServer
133 /* Set the service stack size. This currently just sets the stack
134 * size for all processes to be the maximum seen, so far */
135 #define rx_SetStackSize(service, stackSize) \
136 rx_stackSize = (((stackSize) > rx_stackSize)? stackSize: rx_stackSize)
138 /* Set minimum number of processes guaranteed to be available for this
139 * service at all times */
140 #define rx_SetMinProcs(service, min) ((service)->minProcs = (min))
142 /* Set maximum number of processes that will be made available to this
143 * service (also a guarantee that this number will be made available
144 * if there is no competition) */
145 #define rx_SetMaxProcs(service, max) ((service)->maxProcs = (max))
147 /* Define a procedure to be called just before a server connection is destroyed */
148 #define rx_SetDestroyConnProc(service,proc) ((service)->destroyConnProc = (proc))
150 /* Define procedure to set service dead time */
151 #define rx_SetIdleDeadTime(service,time) ((service)->idleDeadTime = (time))
153 /* Define procedures for getting and setting before and after execute-request procs */
154 #define rx_SetAfterProc(service,proc) ((service)->afterProc = (proc))
155 #define rx_SetBeforeProc(service,proc) ((service)->beforeProc = (proc))
156 #define rx_GetAfterProc(service) ((service)->afterProc)
157 #define rx_GetBeforeProc(service) ((service)->beforeProc)
159 /* Define a procedure to be called when a server connection is created */
160 #define rx_SetNewConnProc(service, proc) ((service)->newConnProc = (proc))
162 /* NOTE: We'll probably redefine the following three routines, again, sometime. */
164 /* Set the connection dead time for any connections created for this service (server only) */
165 #define rx_SetServiceDeadTime(service, seconds) ((service)->secondsUntilDead = (seconds))
167 /* Enable or disable asymmetric client checking for a service */
168 #define rx_SetCheckReach(service, x) ((service)->checkReach = (x))
170 /* Set connection hard and idle timeouts for a connection */
171 #define rx_SetConnHardDeadTime(conn, seconds) ((conn)->hardDeadTime = (seconds))
172 #define rx_SetConnIdleDeadTime(conn, seconds) ((conn)->idleDeadTime = (seconds))
174 /* Set the overload threshold and the overload error */
175 #define rx_SetBusyThreshold(threshold, code) (rx_BusyThreshold=(threshold),rx_BusyError=(code))
177 /* If this flag is set,no new requests are processed by rx, all new requests are
178 returned with an error code of RX_CALL_DEAD ( transient error ) */
179 #define rx_SetRxTranquil() (rx_tranquil = 1)
180 #define rx_ClearRxTranquil() (rx_tranquil = 0)
182 /* Set the threshold and time to delay aborts for consecutive errors */
183 #define rx_SetCallAbortThreshold(A) (rxi_callAbortThreshhold = (A))
184 #define rx_SetCallAbortDelay(A) (rxi_callAbortDelay = (A))
185 #define rx_SetConnAbortThreshold(A) (rxi_connAbortThreshhold = (A))
186 #define rx_SetConnAbortDelay(A) (rxi_connAbortDelay = (A))
188 #define rx_GetCallAbortCode(call) ((call)->abortCode)
189 #define rx_SetCallAbortCode(call, code) ((call)->abortCode = (code))
191 #define cpspace(call) ((call)->curlen)
192 #define cppos(call) ((call)->curpos)
194 #define rx_Read(call, buf, nbytes) rx_ReadProc(call, buf, nbytes)
195 #define rx_Read32(call, value) rx_ReadProc32(call, value)
196 #define rx_Readv(call, iov, nio, maxio, nbytes) \
197 rx_ReadvProc(call, iov, nio, maxio, nbytes)
198 #define rx_Write(call, buf, nbytes) rx_WriteProc(call, buf, nbytes)
199 #define rx_Write32(call, value) rx_WriteProc32(call, value)
200 #define rx_Writev(call, iov, nio, nbytes) \
201 rx_WritevProc(call, iov, nio, nbytes)
203 /* This is the maximum size data packet that can be sent on this connection, accounting for security module-specific overheads. */
204 #define rx_MaxUserDataSize(call) ((call)->MTU - RX_HEADER_SIZE - (call)->conn->securityHeaderSize - (call)->conn->securityMaxTrailerSize)
206 /* Macros to turn the hot thread feature on and off. Enabling hot threads
207 * allows the listener thread to trade places with an idle worker thread,
208 * which moves the context switch from listener to worker out of the
211 #define rx_EnableHotThread() (rx_enable_hot_thread = 1)
212 #define rx_DisableHotThread() (rx_enable_hot_thread = 0)
214 #define rx_PutConnection(conn) rx_DestroyConnection(conn)
216 /* A connection is an authenticated communication path, allowing
217 limited multiple asynchronous conversations. */
219 struct rx_connection_rx_lock {
220 struct rx_connection_rx_lock *next; /* on hash chain _or_ free list */
221 struct rx_peer_rx_lock *peer;
223 struct rx_connection {
224 struct rx_connection *next; /* on hash chain _or_ free list */
225 struct rx_peer *peer;
227 #ifdef RX_ENABLE_LOCKS
228 afs_kmutex_t conn_call_lock; /* locks conn_call_cv */
229 afs_kcondvar_t conn_call_cv;
230 afs_kmutex_t conn_data_lock; /* locks packet data */
232 afs_uint32 epoch; /* Process start time of client side of connection */
233 afs_uint32 cid; /* Connection id (call channel is bottom bits) */
234 afs_int32 error; /* If this connection is in error, this is it */
236 struct rx_call_rx_lock *call[RX_MAXCALLS];
238 struct rx_call *call[RX_MAXCALLS];
240 afs_uint32 callNumber[RX_MAXCALLS]; /* Current call numbers */
241 afs_uint32 serial; /* Next outgoing packet serial number */
242 afs_uint32 lastSerial; /* # of last packet received, for computing skew */
243 afs_int32 maxSerial; /* largest serial number seen on incoming packets */
244 /* afs_int32 maxPacketSize; max packet size should be per-connection since */
245 /* peer process could be restarted on us. Includes RX Header. */
246 struct rxevent *challengeEvent; /* Scheduled when the server is challenging a */
247 struct rxevent *delayedAbortEvent; /* Scheduled to throttle looping client */
248 struct rxevent *checkReachEvent; /* Scheduled when checking reachability */
249 int abortCount; /* count of abort messages sent */
250 /* client-- to retransmit the challenge */
251 struct rx_service *service; /* used by servers only */
252 u_short serviceId; /* To stamp on requests (clients only) */
253 afs_uint32 refCount; /* Reference count */
254 u_char flags; /* Defined below */
255 u_char type; /* Type of connection, defined below */
256 u_char secondsUntilPing; /* how often to ping for each active call */
257 u_char securityIndex; /* corresponds to the security class of the */
258 /* securityObject for this conn */
259 struct rx_securityClass *securityObject; /* Security object for this connection */
260 VOID *securityData; /* Private data for this conn's security class */
261 u_short securityHeaderSize; /* Length of security module's packet header data */
262 u_short securityMaxTrailerSize; /* Length of security module's packet trailer data */
264 int timeout; /* Overall timeout per call (seconds) for this conn */
265 int lastSendTime; /* Last send time for this connection */
266 u_short secondsUntilDead; /* Maximum silence from peer before RX_CALL_DEAD */
267 u_short hardDeadTime; /* hard max for call execution */
268 u_short idleDeadTime; /* max time a call can be idle (no data) */
269 u_char ackRate; /* how many packets between ack requests */
270 u_char makeCallWaiters; /* how many rx_NewCalls are waiting */
271 int nSpecific; /* number entries in specific data */
272 void **specific; /* pointer to connection specific data */
276 /* A service is installed by rx_NewService, and specifies a service type that
277 * is exported by this process. Incoming calls are stamped with the service
278 * type, and must match an installed service for the call to be accepted.
279 * Each service exported has a (port,serviceId) pair to uniquely identify it.
280 * It is also named: this is intended to allow a remote statistics gathering
281 * program to retrieve per service statistics without having to know the local
282 * service id's. Each service has a number of
285 /* security objects (instances of security classes) which implement
286 * various types of end-to-end security protocols for connections made
287 * to this service. Finally, there are two parameters controlling the
288 * number of requests which may be executed in parallel by this
289 * service: minProcs is the number of requests to this service which
290 * are guaranteed to be able to run in parallel at any time; maxProcs
291 * has two meanings: it limits the total number of requests which may
292 * execute in parallel and it also guarantees that that many requests
293 * may be handled in parallel if no other service is handling any
297 u_short serviceId; /* Service number */
298 afs_uint32 serviceHost; /* IP address for this service */
299 u_short servicePort; /* UDP port for this service */
300 char *serviceName; /* Name of the service */
301 osi_socket socket; /* socket structure or file descriptor */
302 u_short nRequestsRunning; /* Number of requests currently in progress */
303 u_short nSecurityObjects; /* Number of entries in security objects array */
304 struct rx_securityClass **securityObjects; /* Array of security class objects */
305 afs_int32(*executeRequestProc) (struct rx_call * acall); /* Routine to call when an rpc request is received */
306 void (*destroyConnProc) (struct rx_connection * tcon); /* Routine to call when a server connection is destroyed */
307 void (*newConnProc) (struct rx_connection * tcon); /* Routine to call when a server connection is created */
308 void (*beforeProc) (struct rx_call * acall); /* routine to call before a call is executed */
309 void (*afterProc) (struct rx_call * acall, afs_int32 code); /* routine to call after a call is executed */
310 u_short maxProcs; /* Maximum procs to be used for this service */
311 u_short minProcs; /* Minimum # of requests guaranteed executable simultaneously */
312 u_short connDeadTime; /* Seconds until a client of this service will be declared dead, if it is not responding */
313 u_short idleDeadTime; /* Time a server will wait for I/O to start up again */
314 u_char checkReach; /* Check for asymmetric clients? */
317 #endif /* KDUMP_RX_LOCK */
319 /* A server puts itself on an idle queue for a service using an
320 * instance of the following structure. When a call arrives, the call
321 * structure pointer is placed in "newcall", the routine to execute to
322 * service the request is placed in executeRequestProc, and the
323 * process is woken up. The queue entry's address is used for the
324 * sleep/wakeup. If socketp is non-null, then this thread is willing
325 * to become a listener thread. A thread sets *socketp to -1 before
326 * sleeping. If *socketp is not -1 when the thread awakes, it is now
327 * the listener thread for *socketp. When socketp is non-null, tno
328 * contains the server's threadID, which is used to make decitions in GetCall.
331 struct rx_serverQueueEntry_rx_lock {
333 struct rx_serverQueueEntry {
335 struct rx_queue queueItemHeader;
337 struct rx_call_rx_lock *newcall;
339 struct rx_call *newcall;
341 #ifdef RX_ENABLE_LOCKS
350 /* A peer refers to a peer process, specified by a (host,port) pair. There may be more than one peer on a given host. */
352 struct rx_peer_rx_lock {
353 struct rx_peer_rx_lock *next; /* Next in hash conflict or free list */
356 struct rx_peer *next; /* Next in hash conflict or free list */
358 #ifdef RX_ENABLE_LOCKS
359 afs_kmutex_t peer_lock; /* Lock peer */
360 #endif /* RX_ENABLE_LOCKS */
361 afs_uint32 host; /* Remote IP address, in net byte order */
362 u_short port; /* Remote UDP port, in net byte order */
364 /* interface mtu probably used for this host - includes RX Header */
365 u_short ifMTU; /* doesn't include IP header */
367 /* For garbage collection */
368 afs_uint32 idleWhen; /* When the refcountwent to zero */
369 afs_uint32 refCount; /* Reference count for this structure */
371 /* Congestion control parameters */
372 u_char burstSize; /* Reinitialization size for the burst parameter */
373 u_char burst; /* Number of packets that can be transmitted right now, without pausing */
374 struct clock burstWait; /* Delay until new burst is allowed */
375 struct rx_queue congestionQueue; /* Calls that are waiting for non-zero burst value */
376 int rtt; /* Round trip time, measured in milliseconds/8 */
377 int rtt_dev; /* rtt smoothed error, in milliseconds/4 */
378 struct clock timeout; /* Current retransmission delay */
379 int nSent; /* Total number of distinct data packets sent, not including retransmissions */
380 int reSends; /* Total number of retransmissions for this peer, since this structure was created */
382 /* Skew: if a packet is received N packets later than expected (based
383 * on packet serial numbers), then we define it to have a skew of N.
384 * The maximum skew values allow us to decide when a packet hasn't
385 * been received yet because it is out-of-order, as opposed to when it
386 * is likely to have been dropped. */
387 afs_uint32 inPacketSkew; /* Maximum skew on incoming packets */
388 afs_uint32 outPacketSkew; /* Peer-reported max skew on our sent packets */
389 int rateFlag; /* Flag for rate testing (-no 0yes +decrement) */
391 /* the "natural" MTU, excluding IP,UDP headers, is negotiated by the endpoints */
394 /* negotiated maximum number of packets to send in a single datagram. */
395 u_short maxDgramPackets;
396 /* local maximum number of packets to send in a single datagram. */
397 u_short ifDgramPackets;
399 * MTU, cwind, and nDgramPackets are used to initialize
400 * slow start parameters for new calls. These values are set whenever a
401 * call sends a retransmission and at the end of each call.
402 * congestSeq is incremented each time the congestion parameters are
403 * changed by a call recovering from a dropped packet. A call used
404 * MAX when updating congestion parameters if it started with the
405 * current congestion sequence number, otherwise it uses MIN.
407 u_short MTU; /* MTU for AFS 3.4a jumboGrams */
408 u_short cwind; /* congestion window */
409 u_short nDgramPackets; /* number packets per AFS 3.5 jumbogram */
410 u_short congestSeq; /* Changed when a call retransmits */
411 afs_hyper_t bytesSent; /* Number of bytes sent to this peer */
412 afs_hyper_t bytesReceived; /* Number of bytes received from this peer */
413 struct rx_queue rpcStats; /* rpc statistic list */
414 int lastReachTime; /* Last time we verified reachability */
418 #ifndef KDUMP_RX_LOCK
419 /* Flag bits for connection structure */
420 #define RX_CONN_MAKECALL_WAITING 1 /* rx_MakeCall is waiting for a channel */
421 #define RX_CONN_DESTROY_ME 2 /* Destroy *client* connection after last call */
422 #define RX_CONN_USING_PACKET_CKSUM 4 /* non-zero header.spare field seen */
423 #define RX_CONN_KNOW_WINDOW 8 /* window size negotiation works */
424 #define RX_CONN_RESET 16 /* connection is reset, remove */
425 #define RX_CONN_BUSY 32 /* connection is busy; don't delete */
426 #define RX_CONN_ATTACHWAIT 64 /* attach waiting for peer->lastReach */
428 /* Type of connection, client or server */
429 #define RX_CLIENT_CONNECTION 0
430 #define RX_SERVER_CONNECTION 1
431 #endif /* !KDUMP_RX_LOCK */
433 /* Call structure: only instantiated for active calls and dallying server calls. The permanent call state (i.e. the call number as well as state shared with other calls associated with this connection) is maintained in the connection structure. */
435 struct rx_call_rx_lock {
439 struct rx_queue queue_item_header; /* Call can be on various queues (one-at-a-time) */
440 struct rx_queue tq; /* Transmit packet queue */
441 struct rx_queue rq; /* Receive packet queue */
443 * The following fields are accessed while the call is unlocked.
444 * These fields are used by the caller/server thread to marshall
445 * and unmarshall RPC data. The only time they may be changed by
446 * other threads is when the RX_CALL_IOVEC_WAIT flag is set.
448 * NOTE: Be sure that these fields start and end on a double
449 * word boundary. Otherwise threads that are changing
450 * adjacent fields will cause problems.
452 struct rx_queue iovq; /* readv/writev packet queue */
453 u_short nLeft; /* Number bytes left in first receive packet */
454 u_short curvec; /* current iovec in currentPacket */
455 u_short curlen; /* bytes remaining in curvec */
456 u_short nFree; /* Number bytes free in last send packet */
457 struct rx_packet *currentPacket; /* Current packet being assembled or being read */
458 char *curpos; /* current position in curvec */
460 * End of fields accessed with call unlocked
462 u_char channel; /* Index of call, within connection */
463 u_char state; /* Current call state as defined below */
464 u_char mode; /* Current mode of a call in ACTIVE state */
465 #ifdef RX_ENABLE_LOCKS
466 afs_kmutex_t lock; /* lock covers data as well as mutexes. */
467 afs_kmutex_t *call_queue_lock; /* points to lock for queue we're on,
469 afs_kcondvar_t cv_twind;
470 afs_kcondvar_t cv_rq;
471 afs_kcondvar_t cv_tq;
474 struct rx_connection_rx_lock *conn; /* Parent connection for call */
476 struct rx_connection *conn; /* Parent connection for this call */
478 afs_uint32 *callNumber; /* Pointer to call number field within connection */
479 afs_uint32 flags; /* Some random flags */
480 u_char localStatus; /* Local user status sent out of band */
481 u_char remoteStatus; /* Remote user status received out of band */
482 afs_int32 error; /* Error condition for this call */
483 afs_uint32 timeout; /* High level timeout for this call */
484 afs_uint32 rnext; /* Next sequence number expected to be read by rx_ReadData */
485 afs_uint32 rprev; /* Previous packet received; used for deciding what the next packet to be received should be, in order to decide whether a negative acknowledge should be sent */
486 afs_uint32 rwind; /* The receive window: the peer must not send packets with sequence numbers >= rnext+rwind */
487 afs_uint32 tfirst; /* First unacknowledged transmit packet number */
488 afs_uint32 tnext; /* Next transmit sequence number to use */
489 u_short twind; /* The transmit window: we cannot assign a sequence number to a packet >= tfirst + twind */
490 u_short cwind; /* The congestion window */
491 u_short nSoftAcked; /* Number soft acked transmit packets */
492 u_short nextCwind; /* The congestion window after recovery */
493 u_short nCwindAcks; /* Number acks received at current cwind */
494 u_short ssthresh; /* The slow start threshold */
495 u_short nDgramPackets; /* Packets per AFS 3.5 jumbogram */
496 u_short nAcks; /* The number of consecttive acks */
497 u_short nNacks; /* Number packets acked that follow the
498 * first negatively acked packet */
499 u_short nSoftAcks; /* The number of delayed soft acks */
500 u_short nHardAcks; /* The number of delayed hard acks */
501 u_short congestSeq; /* Peer's congestion sequence counter */
502 struct rxevent *resendEvent; /* If this is non-Null, there is a retransmission event pending */
503 struct rxevent *timeoutEvent; /* If this is non-Null, then there is an overall timeout for this call */
504 struct rxevent *keepAliveEvent; /* Scheduled periodically in active calls to keep call alive */
505 struct rxevent *delayedAckEvent; /* Scheduled after all packets are received to send an ack if a reply or new call is not generated soon */
506 struct rxevent *delayedAbortEvent; /* Scheduled to throttle looping client */
507 int abortCode; /* error code from last RPC */
508 int abortCount; /* number of times last error was sent */
509 u_int lastSendTime; /* Last time a packet was sent on this call */
510 u_int lastReceiveTime; /* Last time a packet was received for this call */
511 void (*arrivalProc) (register struct rx_call * call, register VOID * mh, register int index); /* Procedure to call when reply is received */
512 VOID *arrivalProcHandle; /* Handle to pass to replyFunc */
513 int arrivalProcArg; /* Additional arg to pass to reply Proc */
514 afs_uint32 lastAcked; /* last packet "hard" acked by receiver */
515 afs_uint32 startWait; /* time server began waiting for input data/send quota */
516 struct clock traceWait; /* time server began waiting for input data/send quota */
517 struct clock traceStart; /* time the call started running */
518 u_short MTU; /* size of packets currently sending */
519 #ifdef RX_ENABLE_LOCKS
520 short refCount; /* Used to keep calls from disappearring
521 * when we get them from a queue. */
522 #endif /* RX_ENABLE_LOCKS */
523 /* Call refcount modifiers */
524 #define RX_CALL_REFCOUNT_BEGIN 0 /* GetCall/NewCall/EndCall */
525 #define RX_CALL_REFCOUNT_RESEND 1 /* resend event */
526 #define RX_CALL_REFCOUNT_DELAY 2 /* delayed ack */
527 #define RX_CALL_REFCOUNT_ALIVE 3 /* keep alive event */
528 #define RX_CALL_REFCOUNT_PACKET 4 /* waiting for packets. */
529 #define RX_CALL_REFCOUNT_SEND 5 /* rxi_Send */
530 #define RX_CALL_REFCOUNT_ACKALL 6 /* rxi_AckAll */
531 #define RX_CALL_REFCOUNT_ABORT 7 /* delayed abort */
532 #define RX_CALL_REFCOUNT_MAX 8 /* array size. */
533 #ifdef RX_REFCOUNT_CHECK
534 short refCDebug[RX_CALL_REFCOUNT_MAX];
535 #endif /* RX_REFCOUNT_CHECK */
536 int iovNBytes; /* byte count for current iovec */
537 int iovMax; /* number elements in current iovec */
538 int iovNext; /* next entry in current iovec */
539 struct iovec *iov; /* current iovec */
540 struct clock queueTime; /* time call was queued */
541 struct clock startTime; /* time call was started */
542 afs_hyper_t bytesSent; /* Number bytes sent */
543 afs_hyper_t bytesRcvd; /* Number bytes received */
547 #ifndef KDUMP_RX_LOCK
548 /* Major call states */
549 #define RX_STATE_NOTINIT 0 /* Call structure has never been initialized */
550 #define RX_STATE_PRECALL 1 /* Server-only: call is not in progress, but packets have arrived */
551 #define RX_STATE_ACTIVE 2 /* An active call; a process is dealing with this call */
552 #define RX_STATE_DALLY 3 /* Dallying after process is done with call */
553 #define RX_STATE_HOLD 4 /* Waiting for acks on reply data packets */
555 /* Call modes: the modes of a call in RX_STATE_ACTIVE state (process attached) */
556 #define RX_MODE_SENDING 1 /* Sending or ready to send */
557 #define RX_MODE_RECEIVING 2 /* Receiving or ready to receive */
558 #define RX_MODE_ERROR 3 /* Something in error for current conversation */
559 #define RX_MODE_EOF 4 /* Server has flushed (or client has read) last reply packet */
562 #define RX_CALL_READER_WAIT 1 /* Reader is waiting for next packet */
563 #define RX_CALL_WAIT_WINDOW_ALLOC 2 /* Sender is waiting for window to allocate buffers */
564 #define RX_CALL_WAIT_WINDOW_SEND 4 /* Sender is waiting for window to send buffers */
565 #define RX_CALL_WAIT_PACKETS 8 /* Sender is waiting for packet buffers */
566 #define RX_CALL_WAIT_PROC 16 /* Waiting for a process to be assigned */
567 #define RX_CALL_RECEIVE_DONE 32 /* All packets received on this call */
568 #define RX_CALL_CLEARED 64 /* Receive queue cleared in precall state */
569 #define RX_CALL_TQ_BUSY 128 /* Call's Xmit Queue is busy; don't modify */
570 #define RX_CALL_TQ_CLEARME 256 /* Need to clear this call's TQ later */
571 #define RX_CALL_TQ_SOME_ACKED 512 /* rxi_Start needs to discard ack'd packets. */
572 #define RX_CALL_TQ_WAIT 1024 /* Reader is waiting for TQ_BUSY to be reset */
573 #define RX_CALL_FAST_RECOVER 2048 /* call is doing congestion recovery */
574 #define RX_CALL_FAST_RECOVER_WAIT 4096 /* thread is waiting to start recovery */
575 #define RX_CALL_SLOW_START_OK 8192 /* receiver acks every other packet */
576 #define RX_CALL_IOVEC_WAIT 16384 /* waiting thread is using an iovec */
577 #define RX_CALL_HAVE_LAST 32768 /* Last packet has been received */
578 #define RX_CALL_NEED_START 0x10000 /* tells rxi_Start to start again */
580 /* Maximum number of acknowledgements in an acknowledge packet */
581 #define RX_MAXACKS 255
583 /* The structure of the data portion of an acknowledge packet: An acknowledge
584 * packet is in network byte order at all times. An acknowledgement is always
585 * prompted for a specific reason by a specific incoming packet. This reason
586 * is reported in "reason" and the packet's sequence number in the packet
587 * header.seq. In addition to this information, all of the current
588 * acknowledgement information about this call is placed in the packet.
589 * "FirstPacket" is the sequence number of the first packet represented in an
590 * array of bytes, "acks", containing acknowledgement information for a number
591 * of consecutive packets. All packets prior to FirstPacket are implicitly
592 * acknowledged: the sender need no longer be concerned about them. Packets
593 * from firstPacket+nAcks and on are not acknowledged. Packets in the range
594 * [firstPacket,firstPacket+nAcks) are each acknowledged explicitly. The
595 * acknowledgement may be RX_NACK if the packet is not (currently) at the
596 * receiver (it may have never been received, or received and then later
597 * dropped), or it may be RX_ACK if the packet is queued up waiting to be read
598 * by the upper level software. RX_ACK does not imply that the packet may not
599 * be dropped before it is read; it does imply that the sender should stop
600 * retransmitting the packet until notified otherwise. The field
601 * previousPacket identifies the previous packet received by the peer. This
602 * was used in a previous version of this software, and could be used in the
603 * future. The serial number in the data part of the ack packet corresponds to
604 * the serial number oof the packet which prompted the acknowledge. Any
605 * packets which are explicitly not acknowledged, and which were last
606 * transmitted with a serial number less than the provided serial number,
607 * should be retransmitted immediately. Actually, this is slightly inaccurate:
608 * packets are not necessarily received in order. When packets are habitually
609 * transmitted out of order, this is allowed for in the retransmission
610 * algorithm by introducing the notion of maximum packet skew: the degree of
611 * out-of-orderness of the packets received on the wire. This number is
612 * communicated from the receiver to the sender in ack packets. */
614 struct rx_ackPacket {
615 u_short bufferSpace; /* Number of packet buffers available. That is: the number of buffers that the sender of the ack packet is willing to provide for data, on this or subsequent calls. Lying is permissable. */
616 u_short maxSkew; /* Maximum difference between serial# of packet acknowledged and highest packet yet received */
617 afs_uint32 firstPacket; /* The first packet in the list of acknowledged packets */
618 afs_uint32 previousPacket; /* The previous packet number received (obsolete?) */
619 afs_uint32 serial; /* Serial number of the packet which prompted the acknowledge */
620 u_char reason; /* Reason for the acknowledge of ackPacket, defined below */
621 u_char nAcks; /* Number of acknowledgements */
622 u_char acks[RX_MAXACKS]; /* Up to RX_MAXACKS packet acknowledgements, defined below */
623 /* Packets <firstPacket are implicitly acknowledged and may be discarded by the sender. Packets >= firstPacket+nAcks are implicitly NOT acknowledged. No packets with sequence numbers >= firstPacket should be discarded by the sender (they may thrown out at any time by the receiver) */
626 #define FIRSTACKOFFSET 4
628 /* Reason for acknowledge message */
629 #define RX_ACK_REQUESTED 1 /* Peer requested an ack on this packet */
630 #define RX_ACK_DUPLICATE 2 /* Duplicate packet */
631 #define RX_ACK_OUT_OF_SEQUENCE 3 /* Packet out of sequence */
632 #define RX_ACK_EXCEEDS_WINDOW 4 /* Packet sequence number higher than window; discarded */
633 #define RX_ACK_NOSPACE 5 /* No buffer space at all */
634 #define RX_ACK_PING 6 /* This is a keep-alive ack */
635 #define RX_ACK_PING_RESPONSE 7 /* Ack'ing because we were pinged */
636 #define RX_ACK_DELAY 8 /* Ack generated since nothing has happened since receiving packet */
637 #define RX_ACK_IDLE 9 /* Similar to RX_ACK_DELAY, but can
640 /* Packet acknowledgement type */
641 #define RX_ACK_TYPE_NACK 0 /* I Don't have this packet */
642 #define RX_ACK_TYPE_ACK 1 /* I have this packet, although I may discard it later */
644 /* The packet size transmitted for an acknowledge is adjusted to reflect the actual size of the acks array. This macro defines the size */
645 #define rx_AckDataSize(nAcks) (3 + nAcks + offsetof(struct rx_ackPacket, acks[0]))
647 #define RX_CHALLENGE_TIMEOUT 2 /* Number of seconds before another authentication request packet is generated */
648 #define RX_CHALLENGE_MAXTRIES 50 /* Max # of times we resend challenge */
649 #define RX_CHECKREACH_TIMEOUT 2 /* Number of seconds before another ping is generated */
650 #define RX_CHECKREACH_TTL 60 /* Re-check reachability this often */
652 /* RX error codes. RX uses error codes from -1 to -64. Rxgen may use other error codes < -64; user programs are expected to return positive error codes */
654 /* Something bad happened to the connection; temporary loss of communication */
655 #define RX_CALL_DEAD (-1)
657 /* An invalid operation, such as a client attempting to send data after having received the beginning of a reply from the server */
658 #define RX_INVALID_OPERATION (-2)
660 /* An optional timeout per call may be specified */
661 #define RX_CALL_TIMEOUT (-3)
663 /* End of data on a read */
666 /* Some sort of low-level protocol error */
667 #define RX_PROTOCOL_ERROR (-5)
669 /* Generic user abort code; used when no more specific error code needs to be communicated. For example, multi rx clients use this code to abort a multi rx call */
670 #define RX_USER_ABORT (-6)
672 /* Port already in use (from rx_Init) */
673 #define RX_ADDRINUSE (-7)
675 /* EMSGSIZE returned from network. Packet too big, must fragment */
676 #define RX_MSGSIZE (-8)
678 /* transient failure detected ( possibly the server is restarting ) */
679 /* this shud be equal to VRESTARTING ( util/errors.h ) for old clients to work */
680 #define RX_RESTARTING (-100)
682 struct rx_securityObjectStats {
683 char type; /* 0:unk 1:null,2:vab 3:kad */
685 char sparec[10]; /* force correct alignment */
686 afs_int32 flags; /* 1=>unalloc, 2=>auth, 4=>expired */
688 afs_uint32 packetsReceived;
689 afs_uint32 packetsSent;
690 afs_uint32 bytesReceived;
691 afs_uint32 bytesSent;
696 /* XXXX (rewrite this description) A security class object contains a set of
697 * procedures and some private data to implement a security model for rx
698 * connections. These routines are called by rx as appropriate. Rx knows
699 * nothing about the internal details of any particular security model, or
700 * about security state. Rx does maintain state per connection on behalf of
701 * the security class. Each security class implementation is also expected to
702 * provide routines to create these objects. Rx provides a basic routine to
703 * allocate one of these objects; this routine must be called by the class. */
704 struct rx_securityClass {
705 struct rx_securityOps {
706 int (*op_Close) (struct rx_securityClass * aobj);
707 int (*op_NewConnection) (struct rx_securityClass * aobj,
708 struct rx_connection * aconn);
709 int (*op_PreparePacket) (struct rx_securityClass * aobj,
710 struct rx_call * acall,
711 struct rx_packet * apacket);
712 int (*op_SendPacket) (struct rx_securityClass * aobj,
713 struct rx_call * acall,
714 struct rx_packet * apacket);
715 int (*op_CheckAuthentication) (struct rx_securityClass * aobj,
716 struct rx_connection * aconn);
717 int (*op_CreateChallenge) (struct rx_securityClass * aobj,
718 struct rx_connection * aconn);
719 int (*op_GetChallenge) (struct rx_securityClass * aobj,
720 struct rx_connection * aconn,
721 struct rx_packet * apacket);
722 int (*op_GetResponse) (struct rx_securityClass * aobj,
723 struct rx_connection * aconn,
724 struct rx_packet * apacket);
725 int (*op_CheckResponse) (struct rx_securityClass * aobj,
726 struct rx_connection * aconn,
727 struct rx_packet * apacket);
728 int (*op_CheckPacket) (struct rx_securityClass * aobj,
729 struct rx_call * acall,
730 struct rx_packet * apacket);
731 int (*op_DestroyConnection) (struct rx_securityClass * aobj,
732 struct rx_connection * aconn);
733 int (*op_GetStats) (struct rx_securityClass * aobj,
734 struct rx_connection * aconn,
735 struct rx_securityObjectStats * astats);
736 int (*op_Spare1) (void);
737 int (*op_Spare2) (void);
738 int (*op_Spare3) (void);
744 #define RXS_OP(obj,op,args) ((obj && (obj->ops->op_ ## op)) ? (*(obj)->ops->op_ ## op)args : 0)
746 #define RXS_Close(obj) RXS_OP(obj,Close,(obj))
747 #define RXS_NewConnection(obj,conn) RXS_OP(obj,NewConnection,(obj,conn))
748 #define RXS_PreparePacket(obj,call,packet) RXS_OP(obj,PreparePacket,(obj,call,packet))
749 #define RXS_SendPacket(obj,call,packet) RXS_OP(obj,SendPacket,(obj,call,packet))
750 #define RXS_CheckAuthentication(obj,conn) RXS_OP(obj,CheckAuthentication,(obj,conn))
751 #define RXS_CreateChallenge(obj,conn) RXS_OP(obj,CreateChallenge,(obj,conn))
752 #define RXS_GetChallenge(obj,conn,packet) RXS_OP(obj,GetChallenge,(obj,conn,packet))
753 #define RXS_GetResponse(obj,conn,packet) RXS_OP(obj,GetResponse,(obj,conn,packet))
754 #define RXS_CheckResponse(obj,conn,packet) RXS_OP(obj,CheckResponse,(obj,conn,packet))
755 #define RXS_CheckPacket(obj,call,packet) RXS_OP(obj,CheckPacket,(obj,call,packet))
756 #define RXS_DestroyConnection(obj,conn) RXS_OP(obj,DestroyConnection,(obj,conn))
757 #define RXS_GetStats(obj,conn,stats) RXS_OP(obj,GetStats,(obj,conn,stats))
761 /* Structure for keeping rx statistics. Note that this structure is returned
762 * by rxdebug, so, for compatibility reasons, new fields should be appended (or
763 * spares used), the rxdebug protocol checked, if necessary, and the PrintStats
764 * code should be updated as well.
766 * Clearly we assume that ntohl will work on these structures so sizeof(int)
767 * must equal sizeof(afs_int32). */
769 struct rx_stats { /* General rx statistics */
770 int packetRequests; /* Number of packet allocation requests */
771 int receivePktAllocFailures;
772 int sendPktAllocFailures;
773 int specialPktAllocFailures;
774 int socketGreedy; /* Whether SO_GREEDY succeeded */
775 int bogusPacketOnRead; /* Number of inappropriately short packets received */
776 int bogusHost; /* Host address from bogus packets */
777 int noPacketOnRead; /* Number of read packets attempted when there was actually no packet to read off the wire */
778 int noPacketBuffersOnRead; /* Number of dropped data packets due to lack of packet buffers */
779 int selects; /* Number of selects waiting for packet or timeout */
780 int sendSelects; /* Number of selects forced when sending packet */
781 int packetsRead[RX_N_PACKET_TYPES]; /* Total number of packets read, per type */
782 int dataPacketsRead; /* Number of unique data packets read off the wire */
783 int ackPacketsRead; /* Number of ack packets read */
784 int dupPacketsRead; /* Number of duplicate data packets read */
785 int spuriousPacketsRead; /* Number of inappropriate data packets */
786 int packetsSent[RX_N_PACKET_TYPES]; /* Number of rxi_Sends: packets sent over the wire, per type */
787 int ackPacketsSent; /* Number of acks sent */
788 int pingPacketsSent; /* Total number of ping packets sent */
789 int abortPacketsSent; /* Total number of aborts */
790 int busyPacketsSent; /* Total number of busies sent received */
791 int dataPacketsSent; /* Number of unique data packets sent */
792 int dataPacketsReSent; /* Number of retransmissions */
793 int dataPacketsPushed; /* Number of retransmissions pushed early by a NACK */
794 int ignoreAckedPacket; /* Number of packets with acked flag, on rxi_Start */
795 struct clock totalRtt; /* Total round trip time measured (use to compute average) */
796 struct clock minRtt; /* Minimum round trip time measured */
797 struct clock maxRtt; /* Maximum round trip time measured */
798 int nRttSamples; /* Total number of round trip samples */
799 int nServerConns; /* Total number of server connections */
800 int nClientConns; /* Total number of client connections */
801 int nPeerStructs; /* Total number of peer structures */
802 int nCallStructs; /* Total number of call structures allocated */
803 int nFreeCallStructs; /* Total number of previously allocated free call structures */
805 afs_int32 fatalErrors;
806 int ignorePacketDally; /* packets dropped because call is in dally state */
807 int receiveCbufPktAllocFailures;
808 int sendCbufPktAllocFailures;
813 /* structures for debug input and output packets */
815 /* debug input types */
821 /* Invalid rx debug package type */
822 #define RX_DEBUGI_BADTYPE (-8)
824 #define RX_DEBUGI_VERSION_MINIMUM ('L') /* earliest real version */
825 #define RX_DEBUGI_VERSION ('R') /* Latest version */
826 /* first version w/ secStats */
827 #define RX_DEBUGI_VERSION_W_SECSTATS ('L')
828 /* version M is first supporting GETALLCONN and RXSTATS type */
829 #define RX_DEBUGI_VERSION_W_GETALLCONN ('M')
830 #define RX_DEBUGI_VERSION_W_RXSTATS ('M')
831 /* last version with unaligned debugConn */
832 #define RX_DEBUGI_VERSION_W_UNALIGNED_CONN ('L')
833 #define RX_DEBUGI_VERSION_W_WAITERS ('N')
834 #define RX_DEBUGI_VERSION_W_IDLETHREADS ('O')
835 #define RX_DEBUGI_VERSION_W_NEWPACKETTYPES ('P')
836 #define RX_DEBUGI_VERSION_W_GETPEER ('Q')
837 #define RX_DEBUGI_VERSION_W_WAITED ('R')
839 #define RX_DEBUGI_GETSTATS 1 /* get basic rx stats */
840 #define RX_DEBUGI_GETCONN 2 /* get connection info */
841 #define RX_DEBUGI_GETALLCONN 3 /* get even uninteresting conns */
842 #define RX_DEBUGI_RXSTATS 4 /* get all rx stats */
843 #define RX_DEBUGI_GETPEER 5 /* get all peer structs */
845 struct rx_debugStats {
846 afs_int32 nFreePackets;
847 afs_int32 packetReclaims;
848 afs_int32 callsExecuted;
849 char waitingForPackets;
854 afs_int32 idleThreads; /* Number of server threads that are idle */
859 struct rx_debugConn_vL {
863 afs_int32 callNumber[RX_MAXCALLS];
869 char callState[RX_MAXCALLS];
870 char callMode[RX_MAXCALLS];
871 char callFlags[RX_MAXCALLS];
872 char callOther[RX_MAXCALLS];
873 /* old style getconn stops here */
874 struct rx_securityObjectStats secStats;
875 afs_int32 sparel[10];
878 struct rx_debugConn {
882 afs_int32 callNumber[RX_MAXCALLS];
888 char sparec[3]; /* force correct alignment */
889 char callState[RX_MAXCALLS];
890 char callMode[RX_MAXCALLS];
891 char callFlags[RX_MAXCALLS];
892 char callOther[RX_MAXCALLS];
893 /* old style getconn stops here */
894 struct rx_securityObjectStats secStats;
900 struct rx_debugPeer {
908 struct clock burstWait;
911 struct clock timeout;
914 afs_int32 inPacketSkew;
915 afs_int32 outPacketSkew;
919 u_short maxDgramPackets;
920 u_short ifDgramPackets;
923 u_short nDgramPackets;
925 afs_hyper_t bytesSent;
926 afs_hyper_t bytesReceived;
927 afs_int32 sparel[10];
930 #define RX_OTHER_IN 1 /* packets avail in in queue */
931 #define RX_OTHER_OUT 2 /* packets avail in out queue */
935 /* Only include this once, even when re-loading for kdump. */
936 #ifndef _CALL_REF_DEFINED_
937 #define _CALL_REF_DEFINED_
939 #ifdef RX_ENABLE_LOCKS
940 #ifdef RX_REFCOUNT_CHECK
941 /* RX_REFCOUNT_CHECK is used to test for call refcount leaks by event
944 extern int rx_callHoldType;
945 #define CALL_HOLD(call, type) do { \
947 call->refCDebug[type]++; \
948 if (call->refCDebug[type] > 50) {\
949 rx_callHoldType = type; \
950 osi_Panic("Huge call refCount"); \
953 #define CALL_RELE(call, type) do { \
955 call->refCDebug[type]--; \
956 if (call->refCDebug[type] > 50) {\
957 rx_callHoldType = type; \
958 osi_Panic("Negative call refCount"); \
961 #else /* RX_REFCOUNT_CHECK */
962 #define CALL_HOLD(call, type) call->refCount++
963 #define CALL_RELE(call, type) call->refCount--
964 #endif /* RX_REFCOUNT_CHECK */
966 #else /* RX_ENABLE_LOCKS */
967 #define CALL_HOLD(call, type)
968 #define CALL_RELE(call, type)
969 #endif /* RX_ENABLE_LOCKS */
971 #endif /* _CALL_REF_DEFINED_ */
973 #define RX_SERVER_DEBUG_SEC_STATS 0x1
974 #define RX_SERVER_DEBUG_ALL_CONN 0x2
975 #define RX_SERVER_DEBUG_RX_STATS 0x4
976 #define RX_SERVER_DEBUG_WAITER_CNT 0x8
977 #define RX_SERVER_DEBUG_IDLE_THREADS 0x10
978 #define RX_SERVER_DEBUG_OLD_CONN 0x20
979 #define RX_SERVER_DEBUG_NEW_PACKETS 0x40
980 #define RX_SERVER_DEBUG_ALL_PEER 0x80
981 #define RX_SERVER_DEBUG_WAITED_CNT 0x100
983 #define AFS_RX_STATS_CLEAR_ALL 0xffffffff
984 #define AFS_RX_STATS_CLEAR_INVOCATIONS 0x1
985 #define AFS_RX_STATS_CLEAR_BYTES_SENT 0x2
986 #define AFS_RX_STATS_CLEAR_BYTES_RCVD 0x4
987 #define AFS_RX_STATS_CLEAR_QUEUE_TIME_SUM 0x8
988 #define AFS_RX_STATS_CLEAR_QUEUE_TIME_SQUARE 0x10
989 #define AFS_RX_STATS_CLEAR_QUEUE_TIME_MIN 0x20
990 #define AFS_RX_STATS_CLEAR_QUEUE_TIME_MAX 0x40
991 #define AFS_RX_STATS_CLEAR_EXEC_TIME_SUM 0x80
992 #define AFS_RX_STATS_CLEAR_EXEC_TIME_SQUARE 0x100
993 #define AFS_RX_STATS_CLEAR_EXEC_TIME_MIN 0x200
994 #define AFS_RX_STATS_CLEAR_EXEC_TIME_MAX 0x400
996 typedef struct rx_function_entry_v1 {
997 afs_uint32 remote_peer;
998 afs_uint32 remote_port;
999 afs_uint32 remote_is_server;
1000 afs_uint32 interfaceId;
1001 afs_uint32 func_total;
1002 afs_uint32 func_index;
1003 afs_hyper_t invocations;
1004 afs_hyper_t bytes_sent;
1005 afs_hyper_t bytes_rcvd;
1006 struct clock queue_time_sum;
1007 struct clock queue_time_sum_sqr;
1008 struct clock queue_time_min;
1009 struct clock queue_time_max;
1010 struct clock execution_time_sum;
1011 struct clock execution_time_sum_sqr;
1012 struct clock execution_time_min;
1013 struct clock execution_time_max;
1014 } rx_function_entry_v1_t, *rx_function_entry_v1_p;
1017 * If you need to change rx_function_entry, you should probably create a brand
1018 * new structure. Keeping the old structure will allow backwards compatibility
1019 * with old clients (even if it is only used to calculate allocation size).
1020 * If you do change the size or the format, you'll need to bump
1021 * RX_STATS_RETRIEVAL_VERSION. This allows some primitive form
1022 * of versioning a la rxdebug.
1025 #define RX_STATS_RETRIEVAL_VERSION 1 /* latest version */
1026 #define RX_STATS_RETRIEVAL_FIRST_EDITION 1 /* first implementation */
1028 typedef struct rx_interface_stat {
1029 struct rx_queue queue_header;
1030 struct rx_queue all_peers;
1031 rx_function_entry_v1_t stats[1]; /* make sure this is aligned correctly */
1032 } rx_interface_stat_t, *rx_interface_stat_p;
1034 #define RX_STATS_SERVICE_ID 409
1038 #endif /* _RX_ End of rx.h */
1041 #include "rx/rx_prototypes.h"
1043 #include "rx_prototypes.h"
1046 #endif /* !KDUMP_RX_LOCK */
git://git.openafs.org / openafs.git / blob