2 * Copyright 2000, International Business Machines Corporation and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
10 /* rx_user.c contains routines specific to the user space UNIX implementation of rx */
12 /* rxi_syscall is currently not prototyped */
14 #include <afsconfig.h>
15 #include <afs/param.h>
22 # include <WINNT/syscfg.h>
26 #if !defined(AFS_AIX_ENV) && !defined(AFS_NT40_ENV)
27 # include <sys/syscall.h>
29 #include <afs/afs_args.h>
30 #include <afs/afsutil.h>
32 #ifndef IPPORT_USERRESERVED
33 /* If in.h doesn't define this, define it anyway. Unfortunately, defining
34 this doesn't put the code into the kernel to restrict kernel assigned
35 port numbers to numbers below IPPORT_USERRESERVED... */
36 #define IPPORT_USERRESERVED 5000
39 #if defined(AFS_LINUX22_ENV) && defined(AFS_RXERRQ_ENV)
40 # include <linux/types.h>
41 # include <linux/errqueue.h>
42 # if defined(AFS_ADAPT_PMTU) && !defined(IP_MTU)
48 #include "rx_atomic.h"
49 #include "rx_globals.h"
52 #include "rx_packet.h"
53 #include "rx_internal.h"
55 #ifdef AFS_PTHREAD_ENV
58 * The rx_if_init_mutex mutex protects the following global variables:
62 afs_kmutex_t rx_if_init_mutex;
63 #define LOCK_IF_INIT MUTEX_ENTER(&rx_if_init_mutex)
64 #define UNLOCK_IF_INIT MUTEX_EXIT(&rx_if_init_mutex)
67 * The rx_if_mutex mutex protects the following global variables:
73 afs_kmutex_t rx_if_mutex;
74 #define LOCK_IF MUTEX_ENTER(&rx_if_mutex)
75 #define UNLOCK_IF MUTEX_EXIT(&rx_if_mutex)
78 #define UNLOCK_IF_INIT
81 #endif /* AFS_PTHREAD_ENV */
85 * Make a socket for receiving/sending IP packets. Set it into non-blocking
86 * and large buffering modes. If port isn't specified, the kernel will pick
87 * one. Returns the socket (>= 0) on success. Returns OSI_NULLSOCKET on
88 * failure. Port must be in network byte order.
91 rxi_GetHostUDPSocket(u_int ahost, u_short port)
94 osi_socket socketFd = OSI_NULLSOCKET;
95 struct sockaddr_in taddr;
96 char *name = "rxi_GetUDPSocket: ";
97 #ifdef AFS_LINUX22_ENV
98 # if defined(AFS_ADAPT_PMTU)
99 int pmtu = IP_PMTUDISC_WANT;
101 int pmtu = IP_PMTUDISC_DONT;
105 #if !defined(AFS_NT40_ENV)
106 if (ntohs(port) >= IPPORT_RESERVED && ntohs(port) < IPPORT_USERRESERVED) {
107 /* (osi_Msg "%s*WARNING* port number %d is not a reserved port number. Use port numbers above %d\n", name, port, IPPORT_USERRESERVED);
110 if (ntohs(port) > 0 && ntohs(port) < IPPORT_RESERVED && geteuid() != 0) {
112 "%sport number %d is a reserved port number which may only be used by root. Use port numbers above %d\n",
113 name, ntohs(port), IPPORT_USERRESERVED);
117 socketFd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
119 if (socketFd == OSI_NULLSOCKET) {
121 fprintf(stderr, "socket() failed with error %u\n", WSAGetLastError());
129 rxi_xmit_init(socketFd);
130 #endif /* AFS_NT40_ENV */
132 taddr.sin_addr.s_addr = ahost;
133 taddr.sin_family = AF_INET;
134 taddr.sin_port = (u_short) port;
135 memset(&taddr.sin_zero, 0, sizeof(taddr.sin_zero));
136 #ifdef STRUCT_SOCKADDR_HAS_SA_LEN
137 taddr.sin_len = sizeof(struct sockaddr_in);
139 #define MAX_RX_BINDS 10
140 for (binds = 0; binds < MAX_RX_BINDS; binds++) {
143 code = bind(socketFd, (struct sockaddr *)&taddr, sizeof(taddr));
147 (osi_Msg "%sbind failed\n", name);
150 #if !defined(AFS_NT40_ENV)
152 * Set close-on-exec on rx socket
154 fcntl(socketFd, F_SETFD, 1);
157 /* Use one of three different ways of getting a socket buffer expanded to
165 len2 = rx_UdpBufSize;
167 /* find the size closest to rx_UdpBufSize that will be accepted */
168 while (!greedy && len2 > len1) {
171 (socketFd, SOL_SOCKET, SO_RCVBUF, (char *)&len2,
177 /* but do not let it get smaller than 32K */
187 (socketFd, SOL_SOCKET, SO_SNDBUF, (char *)&len1,
191 (socketFd, SOL_SOCKET, SO_RCVBUF, (char *)&len2,
194 (osi_Msg "%s*WARNING* Unable to increase buffering on socket\n",
197 rx_atomic_set(&rx_stats.socketGreedy, greedy);
200 #ifdef AFS_LINUX22_ENV
201 setsockopt(socketFd, SOL_IP, IP_MTU_DISCOVER, &pmtu, sizeof(pmtu));
203 #ifdef AFS_RXERRQ_ENV
206 setsockopt(socketFd, SOL_IP, IP_RECVERR, &recverr, sizeof(recverr));
209 if (rxi_Listen(socketFd) < 0) {
217 if (socketFd != OSI_NULLSOCKET)
218 closesocket(socketFd);
220 if (socketFd != OSI_NULLSOCKET)
224 return OSI_NULLSOCKET;
228 rxi_GetUDPSocket(u_short port)
230 return rxi_GetHostUDPSocket(htonl(INADDR_ANY), port);
234 osi_Panic(char *msg, ...)
238 (osi_Msg "Fatal Rx error: ");
247 * osi_AssertFailU() -- used by the osi_Assert() macro.
251 osi_AssertFailU(const char *expr, const char *file, int line)
253 osi_Panic("assertion failed: %s, file: %s, line: %d\n", expr,
257 #if defined(AFS_AIX32_ENV) && !defined(KERNEL)
259 static const char memZero;
261 osi_Alloc(afs_int32 x)
264 * 0-length allocs may return NULL ptr from malloc, so we special-case
265 * things so that NULL returned iff an error occurred
268 return (void *)&memZero;
273 osi_Free(void *x, afs_int32 size)
280 #endif /* defined(AFS_AIX32_ENV) && !defined(KERNEL) */
282 #define ADDRSPERSITE 16
285 static afs_uint32 rxi_NetAddrs[ADDRSPERSITE]; /* host order */
286 static int myNetMTUs[ADDRSPERSITE];
287 static int myNetMasks[ADDRSPERSITE];
288 static int myNetFlags[ADDRSPERSITE];
289 static u_int rxi_numNetAddrs;
290 static int Inited = 0;
292 #if defined(AFS_NT40_ENV)
296 /* The IP address list can change so we must query for it */
299 /* we don't want to use the loopback adapter which is first */
300 /* this is a bad bad hack */
301 if (rxi_numNetAddrs > 1)
302 return htonl(rxi_NetAddrs[1]);
303 else if (rxi_numNetAddrs > 0)
304 return htonl(rxi_NetAddrs[0]);
310 ** return number of addresses
311 ** and the addresses themselves in the buffer
312 ** maxSize - max number of interfaces to return.
315 rx_getAllAddr(afs_uint32 * buffer, int maxSize)
317 int count = 0, offset = 0;
319 /* The IP address list can change so we must query for it */
322 for (count = 0; offset < rxi_numNetAddrs && maxSize > 0;
323 count++, offset++, maxSize--)
324 buffer[count] = htonl(rxi_NetAddrs[offset]);
329 /* this function returns the total number of interface addresses
330 * the buffer has to be passed in by the caller. It also returns
331 * the matching interface mask and mtu. All values are returned
332 * in network byte order.
335 rx_getAllAddrMaskMtu(afs_uint32 addrBuffer[], afs_uint32 maskBuffer[],
336 afs_uint32 mtuBuffer[], int maxSize)
338 int count = 0, offset = 0;
340 /* The IP address list can change so we must query for it */
344 offset < rxi_numNetAddrs && maxSize > 0;
345 count++, offset++, maxSize--) {
346 addrBuffer[count] = htonl(rxi_NetAddrs[offset]);
347 maskBuffer[count] = htonl(myNetMasks[offset]);
348 mtuBuffer[count] = htonl(myNetMTUs[offset]);
355 extern rx_atomic_t rxinit_status;
357 rxi_InitMorePackets(void) {
358 int npackets, ncbufs;
360 ncbufs = (rx_maxJumboRecvSize - RX_FIRSTBUFFERSIZE);
362 ncbufs = ncbufs / RX_CBUFFERSIZE;
363 npackets = rx_initSendWindow - 1;
364 rxi_MorePackets(npackets * (ncbufs + 1));
376 if (Inited < 2 && !rx_atomic_test_bit(&rxinit_status, 0)) {
377 /* We couldn't initialize more packets earlier.
379 rxi_InitMorePackets();
389 rxi_numNetAddrs = ADDRSPERSITE;
390 (void)syscfg_GetIFInfo(&rxi_numNetAddrs, rxi_NetAddrs,
391 myNetMasks, myNetMTUs, myNetFlags);
393 for (i = 0; i < rxi_numNetAddrs; i++) {
394 rxsize = rxi_AdjustIfMTU(myNetMTUs[i] - RX_IPUDP_SIZE);
396 rxi_nRecvFrags * rxsize + (rxi_nRecvFrags - 1) * UDP_HDR_SIZE;
397 maxsize = rxi_AdjustMaxMTU(rxsize, maxsize);
398 if (rx_maxReceiveSize > maxsize) {
399 rx_maxReceiveSize = MIN(RX_MAX_PACKET_SIZE, maxsize);
401 MIN(rx_maxReceiveSize, rx_maxReceiveSizeUser);
403 if (rx_MyMaxSendSize > maxsize) {
404 rx_MyMaxSendSize = MIN(RX_MAX_PACKET_SIZE, maxsize);
410 * If rxinit_status is still set, rx_InitHost() has yet to be called
411 * and we therefore do not have any mutex locks initialized. As a
412 * result we cannot call rxi_MorePackets() without crashing.
414 if (rx_atomic_test_bit(&rxinit_status, 0))
417 rxi_InitMorePackets();
422 fudge_netmask(afs_uint32 addr)
428 else if (IN_CLASSB(addr))
430 else if (IN_CLASSC(addr))
440 #if !defined(AFS_AIX_ENV) && !defined(AFS_NT40_ENV) && !defined(AFS_LINUX20_ENV) && !defined(AFS_DARWIN160_ENV)
442 rxi_syscall(afs_uint32 a3, afs_uint32 a4, void *a5)
447 old = signal(SIGSYS, SIG_IGN);
449 #if defined(AFS_SGI_ENV)
450 rcode = afs_syscall(AFS_SYSCALL, 28, a3, a4, a5);
452 rcode = syscall(AFS_SYSCALL, 28 /* AFSCALL_CALL */ , a3, a4, a5);
453 #endif /* AFS_SGI_ENV */
459 #endif /* AFS_AIX_ENV */
468 struct ifreq ifs[ADDRSPERSITE];
471 char buf[BUFSIZ], *cp, *cplim;
473 struct sockaddr_in *a;
484 memset(rxi_NetAddrs, 0, sizeof(rxi_NetAddrs));
485 memset(myNetFlags, 0, sizeof(myNetFlags));
486 memset(myNetMTUs, 0, sizeof(myNetMTUs));
487 memset(myNetMasks, 0, sizeof(myNetMasks));
489 s = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
490 if (s == OSI_NULLSOCKET)
493 ifc.ifc_len = sizeof(buf);
497 ifc.ifc_len = sizeof(ifs);
498 ifc.ifc_buf = (caddr_t) & ifs[0];
499 memset(&ifs[0], 0, sizeof(ifs));
501 res = ioctl(s, SIOCGIFCONF, &ifc);
503 /* fputs(stderr, "ioctl error IFCONF\n"); */
510 #define size(p) MAX((p).sa_len, sizeof(p))
511 cplim = buf + ifc.ifc_len; /*skip over if's with big ifr_addr's */
512 for (cp = buf; cp < cplim;
513 cp += sizeof(ifr->ifr_name) + MAX(a->sin_len, sizeof(*a))) {
514 if (rxi_numNetAddrs >= ADDRSPERSITE)
517 ifr = (struct ifreq *)cp;
519 len = ifc.ifc_len / sizeof(struct ifreq);
520 if (len > ADDRSPERSITE)
523 for (i = 0; i < len; ++i) {
525 res = ioctl(s, SIOCGIFADDR, ifr);
528 /* fputs(stderr, "ioctl error IFADDR\n");
529 * perror(ifr->ifr_name); */
532 a = (struct sockaddr_in *)&ifr->ifr_addr;
533 if (a->sin_family != AF_INET)
535 rxi_NetAddrs[rxi_numNetAddrs] = ntohl(a->sin_addr.s_addr);
536 if (rx_IsLoopbackAddr(rxi_NetAddrs[rxi_numNetAddrs])) {
537 /* we don't really care about "localhost" */
540 for (j = 0; j < rxi_numNetAddrs; j++) {
541 if (rxi_NetAddrs[j] == rxi_NetAddrs[rxi_numNetAddrs])
544 if (j < rxi_numNetAddrs)
547 /* fprintf(stderr, "if %s addr=%x\n", ifr->ifr_name,
548 * rxi_NetAddrs[rxi_numNetAddrs]); */
551 res = ioctl(s, SIOCGIFFLAGS, ifr);
553 myNetFlags[rxi_numNetAddrs] = ifr->ifr_flags;
555 /* Handle aliased loopbacks as well. */
556 if (ifr->ifr_flags & IFF_LOOPBACK)
559 /* fprintf(stderr, "if %s flags=%x\n",
560 * ifr->ifr_name, ifr->ifr_flags); */
562 * fputs(stderr, "ioctl error IFFLAGS\n");
563 * perror(ifr->ifr_name); */
565 #endif /* SIOCGIFFLAGS */
567 #if !defined(AFS_AIX_ENV) && !defined(AFS_LINUX20_ENV) && !defined(AFS_DARWIN160_ENV)
568 /* this won't run on an AIX system w/o a cache manager */
569 rxi_syscallp = rxi_syscall;
572 /* If I refer to kernel extensions that aren't loaded on AIX, the
573 * program refuses to load and run, so I simply can't include the
574 * following code. Fortunately, AIX is the one operating system in
575 * which the subsequent ioctl works reliably. */
577 if ((*rxi_syscallp) (20 /*AFSOP_GETMTU */ ,
578 htonl(rxi_NetAddrs[rxi_numNetAddrs]),
579 &(myNetMTUs[rxi_numNetAddrs]))) {
580 /* fputs(stderr, "syscall error GETMTU\n");
581 * perror(ifr->ifr_name); */
582 myNetMTUs[rxi_numNetAddrs] = 0;
584 if ((*rxi_syscallp) (42 /*AFSOP_GETMASK */ ,
585 htonl(rxi_NetAddrs[rxi_numNetAddrs]),
586 &(myNetMasks[rxi_numNetAddrs]))) {
587 /* fputs(stderr, "syscall error GETMASK\n");
588 * perror(ifr->ifr_name); */
589 myNetMasks[rxi_numNetAddrs] = 0;
591 myNetMasks[rxi_numNetAddrs] =
592 ntohl(myNetMasks[rxi_numNetAddrs]);
593 /* fprintf(stderr, "if %s mask=0x%x\n",
594 * ifr->ifr_name, myNetMasks[rxi_numNetAddrs]); */
597 if (myNetMTUs[rxi_numNetAddrs] == 0) {
598 myNetMTUs[rxi_numNetAddrs] = OLD_MAX_PACKET_SIZE + RX_IPUDP_SIZE;
600 res = ioctl(s, SIOCGIFMTU, ifr);
601 if ((res == 0) && (ifr->ifr_metric > 128)) { /* sanity check */
602 myNetMTUs[rxi_numNetAddrs] = ifr->ifr_metric;
603 /* fprintf(stderr, "if %s mtu=%d\n",
604 * ifr->ifr_name, ifr->ifr_metric); */
606 /* fputs(stderr, "ioctl error IFMTU\n");
607 * perror(ifr->ifr_name); */
612 if (myNetMasks[rxi_numNetAddrs] == 0) {
613 myNetMasks[rxi_numNetAddrs] =
614 fudge_netmask(rxi_NetAddrs[rxi_numNetAddrs]);
615 #ifdef SIOCGIFNETMASK
616 res = ioctl(s, SIOCGIFNETMASK, ifr);
618 a = (struct sockaddr_in *)&ifr->ifr_addr;
619 myNetMasks[rxi_numNetAddrs] = ntohl(a->sin_addr.s_addr);
620 /* fprintf(stderr, "if %s subnetmask=0x%x\n",
621 * ifr->ifr_name, myNetMasks[rxi_numNetAddrs]); */
623 /* fputs(stderr, "ioctl error IFMASK\n");
624 * perror(ifr->ifr_name); */
629 if (!rx_IsLoopbackAddr(rxi_NetAddrs[rxi_numNetAddrs])) { /* ignore lo0 */
632 rxi_nRecvFrags * (myNetMTUs[rxi_numNetAddrs] - RX_IP_SIZE);
633 maxsize -= UDP_HDR_SIZE; /* only the first frag has a UDP hdr */
634 if (rx_maxReceiveSize < maxsize)
635 rx_maxReceiveSize = MIN(RX_MAX_PACKET_SIZE, maxsize);
642 /* have to allocate at least enough to allow a single packet to reach its
643 * maximum size, so ReadPacket will work. Allocate enough for a couple
644 * of packets to do so, for good measure */
646 int npackets, ncbufs;
648 rx_maxJumboRecvSize =
649 RX_HEADER_SIZE + rxi_nDgramPackets * RX_JUMBOBUFFERSIZE +
650 (rxi_nDgramPackets - 1) * RX_JUMBOHEADERSIZE;
651 rx_maxJumboRecvSize = MAX(rx_maxJumboRecvSize, rx_maxReceiveSize);
652 ncbufs = (rx_maxJumboRecvSize - RX_FIRSTBUFFERSIZE);
654 ncbufs = ncbufs / RX_CBUFFERSIZE;
655 npackets = rx_initSendWindow - 1;
656 rxi_MorePackets(npackets * (ncbufs + 1));
660 #endif /* AFS_NT40_ENV */
662 /* Called from rxi_FindPeer, when initializing a clear rx_peer structure,
663 * to get interesting information.
664 * Curiously enough, the rx_peerHashTable_lock currently protects the
665 * Inited variable (and hence rx_GetIFInfo). When the fs suite uses
666 * pthreads, this issue will need to be revisited.
670 rxi_InitPeerParams(struct rx_peer *pp)
675 #ifdef AFS_ADAPT_PMTU
677 struct sockaddr_in addr;
684 * there's a race here since more than one thread could call
685 * rx_GetIFInfo. The race stops in rx_GetIFInfo.
692 /* try to second-guess IP, and identify which link is most likely to
693 * be used for traffic to/from this host. */
694 ppaddr = ntohl(pp->host);
697 rx_rto_setPeerTimeoutSecs(pp, 2);
698 /* I don't initialize these, because I presume they are bzero'd...
699 * pp->burstSize pp->burst pp->burstWait.sec pp->burstWait.usec
703 for (ix = 0; ix < rxi_numNetAddrs; ++ix) {
704 if ((rxi_NetAddrs[ix] & myNetMasks[ix]) == (ppaddr & myNetMasks[ix])) {
705 #ifdef IFF_POINTOPOINT
706 if (myNetFlags[ix] & IFF_POINTOPOINT)
707 rx_rto_setPeerTimeoutSecs(pp, 4);
708 #endif /* IFF_POINTOPOINT */
710 rxmtu = myNetMTUs[ix] - RX_IPUDP_SIZE;
711 if (rxmtu < RX_MIN_PACKET_SIZE)
712 rxmtu = RX_MIN_PACKET_SIZE;
713 if (pp->ifMTU < rxmtu)
714 pp->ifMTU = MIN(rx_MyMaxSendSize, rxmtu);
718 if (!pp->ifMTU) { /* not local */
719 rx_rto_setPeerTimeoutSecs(pp, 3);
720 pp->ifMTU = MIN(rx_MyMaxSendSize, RX_REMOTE_PACKET_SIZE);
722 #ifdef AFS_ADAPT_PMTU
723 sock=socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
724 if (sock != OSI_NULLSOCKET) {
725 addr.sin_family = AF_INET;
726 addr.sin_addr.s_addr = pp->host;
727 addr.sin_port = pp->port;
728 memset(&addr.sin_zero, 0, sizeof(addr.sin_zero));
729 if (connect(sock, (struct sockaddr *)&addr, sizeof(addr)) == 0) {
731 socklen_t s = sizeof(mtu);
732 if (getsockopt(sock, SOL_IP, IP_MTU, &mtu, &s)== 0) {
733 pp->ifMTU = MIN(mtu - RX_IPUDP_SIZE, pp->ifMTU);
743 pp->ifMTU = rxi_AdjustIfMTU(pp->ifMTU);
744 pp->maxMTU = OLD_MAX_PACKET_SIZE; /* for compatibility with old guys */
745 pp->natMTU = MIN((int)pp->ifMTU, OLD_MAX_PACKET_SIZE);
746 pp->maxDgramPackets =
747 MIN(rxi_nDgramPackets,
748 rxi_AdjustDgramPackets(rxi_nSendFrags, pp->ifMTU));
750 MIN(rxi_nDgramPackets,
751 rxi_AdjustDgramPackets(rxi_nSendFrags, pp->ifMTU));
752 pp->maxDgramPackets = 1;
753 /* Initialize slow start parameters */
754 pp->MTU = MIN(pp->natMTU, pp->maxMTU);
756 pp->nDgramPackets = 1;
760 /* Don't expose jumobgram internals. */
764 rx_maxReceiveSize = OLD_MAX_PACKET_SIZE;
765 rxi_nSendFrags = rxi_nRecvFrags = 1;
768 /* Override max MTU. If rx_SetNoJumbo is called, it must be
769 called before calling rx_SetMaxMTU since SetNoJumbo clobbers rx_maxReceiveSize */
771 rx_SetMaxMTU(int mtu)
773 if (mtu < RX_MIN_PACKET_SIZE || mtu > RX_MAX_PACKET_DATA_SIZE)
776 rx_MyMaxSendSize = rx_maxReceiveSizeUser = rx_maxReceiveSize = mtu;
781 #ifdef AFS_RXERRQ_ENV
783 rxi_HandleSocketError(int socket)
786 struct cmsghdr *cmsg;
787 struct sock_extended_err *err;
788 struct sockaddr_in addr;
789 char controlmsgbuf[256];
792 msg.msg_name = &addr;
793 msg.msg_namelen = sizeof(addr);
796 msg.msg_control = controlmsgbuf;
797 msg.msg_controllen = 256;
799 code = recvmsg(socket, &msg, MSG_ERRQUEUE|MSG_DONTWAIT|MSG_TRUNC);
801 if (code < 0 || !(msg.msg_flags & MSG_ERRQUEUE))
804 for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
805 if (cmsg->cmsg_level == SOL_IP && cmsg->cmsg_type == IP_RECVERR) {
806 err = (struct sock_extended_err *)CMSG_DATA(cmsg);
807 rxi_ProcessNetError(err, addr.sin_addr.s_addr, addr.sin_port);