2 * Copyright 2000, International Business Machines Corporation and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
11 * An implementation of the rx socket listener for pthreads (not using select).
12 * This assumes that multiple read system calls may be extant at any given
13 * time. Also implements the pthread-specific event handling for rx.
15 * rx_pthread.c is used for the thread safe RX package.
18 #include <afsconfig.h>
19 #include <afs/param.h>
24 #include <sys/types.h>
29 # include <sys/socket.h>
30 # include <sys/file.h>
32 # include <netinet/in.h>
34 # include <sys/ioctl.h>
35 # include <sys/time.h>
39 #include <rx/rx_globals.h>
41 #include <rx/rx_pthread.h>
42 #include <rx/rx_clock.h>
45 * Number of times the event handling thread was signalled because a new
46 * event was scheduled earlier than the lastest event.
48 * Protected by event_handler_mutex
50 static long rx_pthread_n_event_wakeups;
52 /* Set rx_pthread_event_rescheduled if event_handler should just try
53 * again instead of sleeping.
55 * Protected by event_handler_mutex
57 static int rx_pthread_event_rescheduled = 0;
59 static void *rx_ListenerProc(void *);
62 * We supply an event handling thread for Rx's event processing.
63 * The condition variable is used to wakeup the thread whenever a new
64 * event is scheduled earlier than the previous earliest event.
65 * This thread is also responsible for keeping time.
67 static pthread_t event_handler_thread;
68 pthread_cond_t rx_event_handler_cond;
69 pthread_mutex_t event_handler_mutex;
70 pthread_cond_t rx_listener_cond;
71 pthread_mutex_t listener_mutex;
72 static int listeners_started = 0;
73 pthread_mutex_t rx_clock_mutex;
74 struct clock rxi_clockNow;
77 * Delay the current thread the specified number of seconds.
86 * Called from rx_Init()
89 rxi_InitializeThreadSupport(void)
91 /* listeners_started must only be reset if
92 * the listener thread terminates */
93 /* listeners_started = 0; */
94 clock_GetTime(&rxi_clockNow);
98 server_entry(void *argp)
100 void (*server_proc) () = (void (*)())argp;
102 dpf(("rx_pthread.c: server_entry: Server proc returned unexpectedly\n"));
108 * Start an Rx server process.
111 rxi_StartServerProc(void (*proc) (void), int stacksize)
114 pthread_attr_t tattr;
117 if (pthread_attr_init(&tattr) != 0) {
118 dpf(("Unable to Create Rx server thread (pthread_attr_init)\n"));
122 if (pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED) != 0) {
124 (("Unable to Create Rx server thread (pthread_attr_setdetachstate)\n"));
129 * NOTE: We are ignoring the stack size parameter, for now.
132 if (pthread_create(&thread, &tattr, server_entry, (void *)proc) != 0) {
133 dpf(("Unable to Create Rx server thread\n"));
136 AFS_SIGSET_RESTORE();
140 * The event handling process.
143 event_handler(void *argp)
145 struct clock rx_pthread_last_event_wait_time = { 0, 0 };
146 unsigned long rx_pthread_n_event_expired = 0;
147 unsigned long rx_pthread_n_event_waits = 0;
148 long rx_pthread_n_event_woken = 0;
149 unsigned long rx_pthread_n_event_error = 0;
150 struct timespec rx_pthread_next_event_time = { 0, 0 };
153 assert(pthread_mutex_lock(&event_handler_mutex) == 0);
159 assert(pthread_mutex_unlock(&event_handler_mutex) == 0);
161 next.sec = 30; /* Time to sleep if there are no events scheduled */
164 rxevent_RaiseEvents(&next);
166 assert(pthread_mutex_lock(&event_handler_mutex) == 0);
167 if (rx_pthread_event_rescheduled) {
168 rx_pthread_event_rescheduled = 0;
172 clock_Add(&cv, &next);
173 rx_pthread_next_event_time.tv_sec = cv.sec;
174 rx_pthread_next_event_time.tv_nsec = cv.usec * 1000;
175 rx_pthread_n_event_waits++;
176 error = pthread_cond_timedwait
177 (&rx_event_handler_cond, &event_handler_mutex,
178 &rx_pthread_next_event_time);
180 rx_pthread_n_event_woken++;
183 else if (error == ETIMEDOUT) {
184 rx_pthread_n_event_expired++;
186 rx_pthread_n_event_error++;
189 else if (errno == ETIMEDOUT) {
190 rx_pthread_n_event_expired++;
192 rx_pthread_n_event_error++;
195 rx_pthread_event_rescheduled = 0;
201 * This routine will get called by the event package whenever a new,
202 * earlier than others, event is posted. */
204 rxi_ReScheduleEvents(void)
206 assert(pthread_mutex_lock(&event_handler_mutex) == 0);
207 pthread_cond_signal(&rx_event_handler_cond);
208 rx_pthread_event_rescheduled = 1;
209 assert(pthread_mutex_unlock(&event_handler_mutex) == 0);
213 /* Loop to listen on a socket. Return setting *newcallp if this
214 * thread should become a server thread. */
216 rxi_ListenerProc(osi_socket sock, int *tnop, struct rx_call **newcallp)
220 register struct rx_packet *p = (struct rx_packet *)0;
222 assert(pthread_mutex_lock(&listener_mutex) == 0);
223 while (!listeners_started) {
224 assert(pthread_cond_wait(&rx_listener_cond, &listener_mutex) == 0);
226 assert(pthread_mutex_unlock(&listener_mutex) == 0);
230 * Grab a new packet only if necessary (otherwise re-use the old one)
233 rxi_RestoreDataBufs(p);
235 if (!(p = rxi_AllocPacket(RX_PACKET_CLASS_RECEIVE))) {
236 /* Could this happen with multiple socket listeners? */
237 dpf(("rxi_Listener: no packets!")); /* Shouldn't happen */
242 if (rxi_ReadPacket(sock, p, &host, &port)) {
244 p = rxi_ReceivePacket(p, sock, host, port, tnop, newcallp);
245 if (newcallp && *newcallp) {
255 /* This is the listener process request loop. The listener process loop
256 * becomes a server thread when rxi_ListenerProc returns, and stays
257 * server thread until rxi_ServerProc returns. */
259 rx_ListenerProc(void *argp)
262 osi_socket sock = (osi_socket)argp;
263 struct rx_call *newcall;
268 rxi_ListenerProc(sock, &threadID, &newcall);
269 /* assert(threadID != -1); */
270 /* assert(newcall != NULL); */
271 sock = OSI_NULLSOCKET;
272 assert(pthread_setspecific(rx_thread_id_key, (void *)threadID) == 0);
273 rxi_ServerProc(threadID, newcall, &sock);
274 /* assert(sock != OSI_NULLSOCKET); */
279 /* This is the server process request loop. The server process loop
280 * becomes a listener thread when rxi_ServerProc returns, and stays
281 * listener thread until rxi_ListenerProc returns. */
287 struct rx_call *newcall = NULL;
289 rxi_MorePackets(rx_maxReceiveWindow + 2); /* alloc more packets */
290 MUTEX_ENTER(&rx_stats_mutex);
291 rxi_dataQuota += rx_initSendWindow; /* Reserve some pkts for hard times */
292 /* threadID is used for making decisions in GetCall. Get it by bumping
293 * number of threads handling incoming calls */
294 /* Unique thread ID: used for scheduling purposes *and* as index into
295 * the host hold table (fileserver).
296 * The previously used rxi_availProcs is unsuitable as it
297 * will already go up and down as packets arrive while the server
298 * threads are still initialising! The recently introduced
299 * rxi_pthread_hinum does not necessarily lead to a server
300 * thread with id 0, which is not allowed to hop through the
301 * incoming call queue.
302 * So either introduce yet another counter or flag the FCFS
303 * thread... chose the latter.
305 threadID = ++rxi_pthread_hinum;
306 if (rxi_fcfs_thread_num == 0 && rxi_fcfs_thread_num != threadID)
307 rxi_fcfs_thread_num = threadID;
309 MUTEX_EXIT(&rx_stats_mutex);
312 sock = OSI_NULLSOCKET;
313 assert(pthread_setspecific(rx_thread_id_key, (void *)threadID) == 0);
314 rxi_ServerProc(threadID, newcall, &sock);
315 /* assert(sock != OSI_NULLSOCKET); */
317 rxi_ListenerProc(sock, &threadID, &newcall);
318 /* assert(threadID != -1); */
319 /* assert(newcall != NULL); */
325 * Historically used to start the listener process. We now have multiple
326 * listener processes (one for each socket); these are started by GetUdpSocket.
328 * The event handling process *is* started here (the old listener used
329 * to also handle events). The listener threads can't actually start
330 * listening until rxi_StartListener is called because most of R may not
331 * be initialized when rxi_Listen is called.
334 rxi_StartListener(void)
336 pthread_attr_t tattr;
339 if (listeners_started)
342 if (pthread_attr_init(&tattr) != 0) {
344 (("Unable to create Rx event handling thread (pthread_attr_init)\n"));
348 if (pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED) != 0) {
350 (("Unable to create Rx event handling thread (pthread_attr_setdetachstate)\n"));
355 if (pthread_create(&event_handler_thread, &tattr, event_handler, NULL) !=
357 dpf(("Unable to create Rx event handling thread\n"));
360 MUTEX_ENTER(&rx_stats_mutex);
362 MUTEX_EXIT(&rx_stats_mutex);
363 AFS_SIGSET_RESTORE();
365 assert(pthread_mutex_lock(&listener_mutex) == 0);
366 assert(pthread_cond_broadcast(&rx_listener_cond) == 0);
367 listeners_started = 1;
368 assert(pthread_mutex_unlock(&listener_mutex) == 0);
373 * Listen on the specified socket.
376 rxi_Listen(osi_socket sock)
379 pthread_attr_t tattr;
382 if (pthread_attr_init(&tattr) != 0) {
384 (("Unable to create socket listener thread (pthread_attr_init)\n"));
388 if (pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED) != 0) {
390 (("Unable to create socket listener thread (pthread_attr_setdetachstate)\n"));
395 if (pthread_create(&thread, &tattr, rx_ListenerProc, (void *)sock) != 0) {
396 dpf(("Unable to create socket listener thread\n"));
399 MUTEX_ENTER(&rx_stats_mutex);
401 MUTEX_EXIT(&rx_stats_mutex);
402 AFS_SIGSET_RESTORE();
412 rxi_Recvmsg(osi_socket socket, struct msghdr *msg_p, int flags)
415 ret = recvmsg(socket, msg_p, flags);
423 rxi_Sendmsg(osi_socket socket, struct msghdr *msg_p, int flags)
426 ret = sendmsg(socket, msg_p, flags);
427 #ifdef AFS_LINUX22_ENV
428 /* linux unfortunately returns ECONNREFUSED if the target port
429 * is no longer in use */
430 /* and EAGAIN if a UDP checksum is incorrect */
431 if (ret == -1 && errno != ECONNREFUSED && errno != EAGAIN) {
435 dpf(("rxi_sendmsg failed, error %d\n", errno));
442 struct rx_ts_info_t * rx_ts_info_init() {
443 register struct rx_ts_info_t * rx_ts_info;
444 rx_ts_info = (rx_ts_info_t *) malloc(sizeof(rx_ts_info_t));
445 assert(rx_ts_info != NULL && pthread_setspecific(rx_ts_info_key, rx_ts_info) == 0);
446 memset(rx_ts_info, 0, sizeof(rx_ts_info_t));
447 #ifdef RX_ENABLE_TSFPQ
448 queue_Init(&rx_ts_info->_FPQ);
450 MUTEX_ENTER(&rx_stats_mutex);
452 RX_TS_FPQ_COMPUTE_LIMITS;
453 MUTEX_EXIT(&rx_stats_mutex);
454 #endif /* RX_ENABLE_TSFPQ */