2 * Copyright 2000, International Business Machines Corporation and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
10 /* rx_user.c contains routines specific to the user space UNIX implementation of rx */
12 #include <afsconfig.h>
13 #include <afs/param.h>
18 # include <sys/types.h>
23 # include <WINNT/syscfg.h>
25 # include <sys/socket.h>
26 # include <sys/file.h>
28 # include <sys/stat.h>
29 # include <netinet/in.h>
30 # include <sys/time.h>
32 # include <sys/ioctl.h>
35 #if !defined(AFS_AIX_ENV) && !defined(AFS_NT40_ENV)
36 # include <sys/syscall.h>
38 #include <afs/afs_args.h>
39 #include <afs/afsutil.h>
41 #ifndef IPPORT_USERRESERVED
42 /* If in.h doesn't define this, define it anyway. Unfortunately, defining
43 this doesn't put the code into the kernel to restrict kernel assigned
44 port numbers to numbers below IPPORT_USERRESERVED... */
45 #define IPPORT_USERRESERVED 5000
49 # include <sys/time.h>
51 # include "rx_internal.h"
53 # include "rx_globals.h"
55 #ifdef AFS_PTHREAD_ENV
59 * The rx_if_init_mutex mutex protects the following global variables:
63 afs_kmutex_t rx_if_init_mutex;
64 #define LOCK_IF_INIT MUTEX_ENTER(&rx_if_init_mutex)
65 #define UNLOCK_IF_INIT MUTEX_EXIT(&rx_if_init_mutex)
68 * The rx_if_mutex mutex protects the following global variables:
74 afs_kmutex_t rx_if_mutex;
75 #define LOCK_IF MUTEX_ENTER(&rx_if_mutex)
76 #define UNLOCK_IF MUTEX_EXIT(&rx_if_mutex)
79 #define UNLOCK_IF_INIT
82 #endif /* AFS_PTHREAD_ENV */
86 * Make a socket for receiving/sending IP packets. Set it into non-blocking
87 * and large buffering modes. If port isn't specified, the kernel will pick
88 * one. Returns the socket (>= 0) on success. Returns OSI_NULLSOCKET on
89 * failure. Port must be in network byte order.
92 rxi_GetHostUDPSocket(u_int ahost, u_short port)
95 osi_socket socketFd = OSI_NULLSOCKET;
96 struct sockaddr_in taddr;
97 char *name = "rxi_GetUDPSocket: ";
98 #ifdef AFS_LINUX22_ENV
99 #if defined(ADAPT_PMTU)
100 int pmtu=IP_PMTUDISC_WANT;
103 int pmtu=IP_PMTUDISC_DONT;
106 #if defined(HAVE_LINUX_ERRQUEUE_H) && defined(ADAPT_PMTU)
107 #include <linux/types.h>
108 #include <linux/errqueue.h>
114 #if !defined(AFS_NT40_ENV)
115 if (ntohs(port) >= IPPORT_RESERVED && ntohs(port) < IPPORT_USERRESERVED) {
116 /* (osi_Msg "%s*WARNING* port number %d is not a reserved port number. Use port numbers above %d\n", name, port, IPPORT_USERRESERVED);
119 if (ntohs(port) > 0 && ntohs(port) < IPPORT_RESERVED && geteuid() != 0) {
121 "%sport number %d is a reserved port number which may only be used by root. Use port numbers above %d\n",
122 name, ntohs(port), IPPORT_USERRESERVED);
126 socketFd = socket(AF_INET, SOCK_DGRAM, 0);
134 rxi_xmit_init(socketFd);
135 #endif /* AFS_NT40_ENV */
137 taddr.sin_addr.s_addr = ahost;
138 taddr.sin_family = AF_INET;
139 taddr.sin_port = (u_short) port;
140 #ifdef STRUCT_SOCKADDR_HAS_SA_LEN
141 taddr.sin_len = sizeof(struct sockaddr_in);
143 #define MAX_RX_BINDS 10
144 for (binds = 0; binds < MAX_RX_BINDS; binds++) {
147 code = bind(socketFd, (struct sockaddr *)&taddr, sizeof(taddr));
153 (osi_Msg "%sbind failed\n", name);
156 #if !defined(AFS_NT40_ENV)
158 * Set close-on-exec on rx socket
160 fcntl(socketFd, F_SETFD, 1);
163 /* Use one of three different ways of getting a socket buffer expanded to
171 len2 = rx_UdpBufSize;
173 /* find the size closest to rx_UdpBufSize that will be accepted */
174 while (!greedy && len2 > len1) {
177 (socketFd, SOL_SOCKET, SO_RCVBUF, (char *)&len2,
183 /* but do not let it get smaller than 32K */
193 (socketFd, SOL_SOCKET, SO_SNDBUF, (char *)&len1,
197 (socketFd, SOL_SOCKET, SO_RCVBUF, (char *)&len2,
200 (osi_Msg "%s*WARNING* Unable to increase buffering on socket\n",
202 if (rx_stats_active) {
203 MUTEX_ENTER(&rx_stats_mutex);
204 rx_stats.socketGreedy = greedy;
205 MUTEX_EXIT(&rx_stats_mutex);
209 #ifdef AFS_LINUX22_ENV
210 setsockopt(socketFd, SOL_IP, IP_MTU_DISCOVER, &pmtu, sizeof(pmtu));
211 #if defined(ADAPT_PMTU)
212 setsockopt(socketFd, SOL_IP, IP_RECVERR, &recverr, sizeof(recverr));
215 if (rxi_Listen(socketFd) < 0) {
224 closesocket(socketFd);
230 return OSI_NULLSOCKET;
234 rxi_GetUDPSocket(u_short port)
236 return rxi_GetHostUDPSocket(htonl(INADDR_ANY), port);
240 osi_Panic(char *msg, ...)
244 (osi_Msg "Fatal Rx error: ");
253 * osi_AssertFailU() -- used by the osi_Assert() macro.
257 osi_AssertFailU(const char *expr, const char *file, int line)
259 osi_Panic("assertion failed: %s, file: %s, line: %d\n", expr,
263 #if defined(AFS_AIX32_ENV) && !defined(KERNEL)
265 static const char memZero;
267 osi_Alloc(afs_int32 x)
270 * 0-length allocs may return NULL ptr from malloc, so we special-case
271 * things so that NULL returned iff an error occurred
274 return (void *)&memZero;
279 osi_Free(void *x, afs_int32 size)
286 #endif /* defined(AFS_AIX32_ENV) && !defined(KERNEL) */
288 #define ADDRSPERSITE 16
291 static afs_uint32 rxi_NetAddrs[ADDRSPERSITE]; /* host order */
292 static int myNetMTUs[ADDRSPERSITE];
293 static int myNetMasks[ADDRSPERSITE];
294 static int myNetFlags[ADDRSPERSITE];
295 static u_int rxi_numNetAddrs;
296 static int Inited = 0;
298 #if defined(AFS_NT40_ENV)
302 /* The IP address list can change so we must query for it */
305 /* we don't want to use the loopback adapter which is first */
306 /* this is a bad bad hack */
307 if (rxi_numNetAddrs > 1)
308 return htonl(rxi_NetAddrs[1]);
309 else if (rxi_numNetAddrs > 0)
310 return htonl(rxi_NetAddrs[0]);
316 ** return number of addresses
317 ** and the addresses themselves in the buffer
318 ** maxSize - max number of interfaces to return.
321 rx_getAllAddr(afs_uint32 * buffer, int maxSize)
323 int count = 0, offset = 0;
325 /* The IP address list can change so we must query for it */
329 /* we don't want to use the loopback adapter which is first */
330 /* this is a bad bad hack.
331 * and doesn't hold true on Windows.
333 if ( rxi_numNetAddrs > 1 )
335 #endif /* AFS_DJGPP_ENV */
337 for (count = 0; offset < rxi_numNetAddrs && maxSize > 0;
338 count++, offset++, maxSize--)
339 buffer[count] = htonl(rxi_NetAddrs[offset]);
344 /* this function returns the total number of interface addresses
345 * the buffer has to be passed in by the caller. It also returns
346 * the matching interface mask and mtu. All values are returned
347 * in network byte order.
350 rx_getAllAddrMaskMtu(afs_uint32 addrBuffer[], afs_uint32 maskBuffer[],
351 afs_uint32 mtuBuffer[], int maxSize)
353 int count = 0, offset = 0;
355 /* The IP address list can change so we must query for it */
359 /* we don't want to use the loopback adapter which is first */
360 /* this is a bad bad hack.
361 * and doesn't hold true on Windows.
363 if ( rxi_numNetAddrs > 1 )
365 #endif /* AFS_DJGPP_ENV */
368 offset < rxi_numNetAddrs && maxSize > 0;
369 count++, offset++, maxSize--) {
370 addrBuffer[count] = htonl(rxi_NetAddrs[offset]);
371 maskBuffer[count] = htonl(myNetMasks[offset]);
372 mtuBuffer[count] = htonl(myNetMTUs[offset]);
379 extern int rxinit_status;
381 rxi_InitMorePackets(void) {
382 int npackets, ncbufs;
384 ncbufs = (rx_maxJumboRecvSize - RX_FIRSTBUFFERSIZE);
386 ncbufs = ncbufs / RX_CBUFFERSIZE;
387 npackets = rx_initSendWindow - 1;
388 rxi_MorePackets(npackets * (ncbufs + 1));
400 if (Inited < 2 && rxinit_status == 0) {
401 /* We couldn't initialize more packets earlier.
403 rxi_InitMorePackets();
413 rxi_numNetAddrs = ADDRSPERSITE;
414 (void)syscfg_GetIFInfo(&rxi_numNetAddrs, rxi_NetAddrs,
415 myNetMasks, myNetMTUs, myNetFlags);
417 for (i = 0; i < rxi_numNetAddrs; i++) {
418 rxsize = rxi_AdjustIfMTU(myNetMTUs[i] - RX_IPUDP_SIZE);
420 rxi_nRecvFrags * rxsize + (rxi_nRecvFrags - 1) * UDP_HDR_SIZE;
421 maxsize = rxi_AdjustMaxMTU(rxsize, maxsize);
422 if (rx_maxReceiveSize < maxsize) {
423 rx_maxReceiveSize = MIN(RX_MAX_PACKET_SIZE, maxsize);
425 MIN(rx_maxReceiveSize, rx_maxReceiveSizeUser);
432 * If rxinit_status is still set, rx_InitHost() has yet to be called
433 * and we therefore do not have any mutex locks initialized. As a
434 * result we cannot call rxi_MorePackets() without crashing.
439 rxi_InitMorePackets();
444 fudge_netmask(afs_uint32 addr)
450 else if (IN_CLASSB(addr))
452 else if (IN_CLASSC(addr))
462 #if !defined(AFS_AIX_ENV) && !defined(AFS_NT40_ENV) && !defined(AFS_LINUX20_ENV)
464 rxi_syscall(a3, a4, a5)
471 old = (void (*)())signal(SIGSYS, SIG_IGN);
473 #if defined(AFS_SGI_ENV)
474 rcode = afs_syscall(a3, a4, a5);
476 rcode = syscall(AFS_SYSCALL, 28 /* AFSCALL_CALL */ , a3, a4, a5);
477 #endif /* AFS_SGI_ENV */
483 #endif /* AFS_AIX_ENV */
492 struct ifreq ifs[ADDRSPERSITE];
495 char buf[BUFSIZ], *cp, *cplim;
497 struct sockaddr_in *a;
508 memset(rxi_NetAddrs, 0, sizeof(rxi_NetAddrs));
509 memset(myNetFlags, 0, sizeof(myNetFlags));
510 memset(myNetMTUs, 0, sizeof(myNetMTUs));
511 memset(myNetMasks, 0, sizeof(myNetMasks));
513 s = socket(AF_INET, SOCK_DGRAM, 0);
518 ifc.ifc_len = sizeof(buf);
522 ifc.ifc_len = sizeof(ifs);
523 ifc.ifc_buf = (caddr_t) & ifs[0];
524 memset(&ifs[0], 0, sizeof(ifs));
526 res = ioctl(s, SIOCGIFCONF, &ifc);
528 /* fputs(stderr, "ioctl error IFCONF\n"); */
535 #define size(p) MAX((p).sa_len, sizeof(p))
536 cplim = buf + ifc.ifc_len; /*skip over if's with big ifr_addr's */
537 for (cp = buf; cp < cplim;
538 cp += sizeof(ifr->ifr_name) + MAX(a->sin_len, sizeof(*a))) {
539 if (rxi_numNetAddrs >= ADDRSPERSITE)
542 ifr = (struct ifreq *)cp;
544 len = ifc.ifc_len / sizeof(struct ifreq);
545 if (len > ADDRSPERSITE)
548 for (i = 0; i < len; ++i) {
550 res = ioctl(s, SIOCGIFADDR, ifr);
553 /* fputs(stderr, "ioctl error IFADDR\n");
554 * perror(ifr->ifr_name); */
557 a = (struct sockaddr_in *)&ifr->ifr_addr;
558 if (a->sin_family != AF_INET)
560 rxi_NetAddrs[rxi_numNetAddrs] = ntohl(a->sin_addr.s_addr);
561 if (rxi_NetAddrs[rxi_numNetAddrs] == 0x7f000001) {
562 /* we don't really care about "localhost" */
565 for (j = 0; j < rxi_numNetAddrs; j++) {
566 if (rxi_NetAddrs[j] == rxi_NetAddrs[rxi_numNetAddrs])
569 if (j < rxi_numNetAddrs)
572 /* fprintf(stderr, "if %s addr=%x\n", ifr->ifr_name,
573 * rxi_NetAddrs[rxi_numNetAddrs]); */
576 res = ioctl(s, SIOCGIFFLAGS, ifr);
578 myNetFlags[rxi_numNetAddrs] = ifr->ifr_flags;
580 /* Handle aliased loopbacks as well. */
581 if (ifr->ifr_flags & IFF_LOOPBACK)
584 /* fprintf(stderr, "if %s flags=%x\n",
585 * ifr->ifr_name, ifr->ifr_flags); */
587 * fputs(stderr, "ioctl error IFFLAGS\n");
588 * perror(ifr->ifr_name); */
590 #endif /* SIOCGIFFLAGS */
592 #if !defined(AFS_AIX_ENV) && !defined(AFS_LINUX20_ENV)
593 /* this won't run on an AIX system w/o a cache manager */
594 rxi_syscallp = rxi_syscall;
597 /* If I refer to kernel extensions that aren't loaded on AIX, the
598 * program refuses to load and run, so I simply can't include the
599 * following code. Fortunately, AIX is the one operating system in
600 * which the subsequent ioctl works reliably. */
602 if ((*rxi_syscallp) (20 /*AFSOP_GETMTU */ ,
603 htonl(rxi_NetAddrs[rxi_numNetAddrs]),
604 &(myNetMTUs[rxi_numNetAddrs]))) {
605 /* fputs(stderr, "syscall error GETMTU\n");
606 * perror(ifr->ifr_name); */
607 myNetMTUs[rxi_numNetAddrs] = 0;
609 if ((*rxi_syscallp) (42 /*AFSOP_GETMASK */ ,
610 htonl(rxi_NetAddrs[rxi_numNetAddrs]),
611 &(myNetMasks[rxi_numNetAddrs]))) {
612 /* fputs(stderr, "syscall error GETMASK\n");
613 * perror(ifr->ifr_name); */
614 myNetMasks[rxi_numNetAddrs] = 0;
616 myNetMasks[rxi_numNetAddrs] =
617 ntohl(myNetMasks[rxi_numNetAddrs]);
618 /* fprintf(stderr, "if %s mask=0x%x\n",
619 * ifr->ifr_name, myNetMasks[rxi_numNetAddrs]); */
622 if (myNetMTUs[rxi_numNetAddrs] == 0) {
623 myNetMTUs[rxi_numNetAddrs] = OLD_MAX_PACKET_SIZE + RX_IPUDP_SIZE;
625 res = ioctl(s, SIOCGIFMTU, ifr);
626 if ((res == 0) && (ifr->ifr_metric > 128)) { /* sanity check */
627 myNetMTUs[rxi_numNetAddrs] = ifr->ifr_metric;
628 /* fprintf(stderr, "if %s mtu=%d\n",
629 * ifr->ifr_name, ifr->ifr_metric); */
631 /* fputs(stderr, "ioctl error IFMTU\n");
632 * perror(ifr->ifr_name); */
637 if (myNetMasks[rxi_numNetAddrs] == 0) {
638 myNetMasks[rxi_numNetAddrs] =
639 fudge_netmask(rxi_NetAddrs[rxi_numNetAddrs]);
640 #ifdef SIOCGIFNETMASK
641 res = ioctl(s, SIOCGIFNETMASK, ifr);
643 a = (struct sockaddr_in *)&ifr->ifr_addr;
644 myNetMasks[rxi_numNetAddrs] = ntohl(a->sin_addr.s_addr);
645 /* fprintf(stderr, "if %s subnetmask=0x%x\n",
646 * ifr->ifr_name, myNetMasks[rxi_numNetAddrs]); */
648 /* fputs(stderr, "ioctl error IFMASK\n");
649 * perror(ifr->ifr_name); */
654 if (rxi_NetAddrs[rxi_numNetAddrs] != 0x7f000001) { /* ignore lo0 */
657 rxi_nRecvFrags * (myNetMTUs[rxi_numNetAddrs] - RX_IP_SIZE);
658 maxsize -= UDP_HDR_SIZE; /* only the first frag has a UDP hdr */
659 if (rx_maxReceiveSize < maxsize)
660 rx_maxReceiveSize = MIN(RX_MAX_PACKET_SIZE, maxsize);
667 /* have to allocate at least enough to allow a single packet to reach its
668 * maximum size, so ReadPacket will work. Allocate enough for a couple
669 * of packets to do so, for good measure */
671 int npackets, ncbufs;
673 rx_maxJumboRecvSize =
674 RX_HEADER_SIZE + rxi_nDgramPackets * RX_JUMBOBUFFERSIZE +
675 (rxi_nDgramPackets - 1) * RX_JUMBOHEADERSIZE;
676 rx_maxJumboRecvSize = MAX(rx_maxJumboRecvSize, rx_maxReceiveSize);
677 ncbufs = (rx_maxJumboRecvSize - RX_FIRSTBUFFERSIZE);
679 ncbufs = ncbufs / RX_CBUFFERSIZE;
680 npackets = rx_initSendWindow - 1;
681 rxi_MorePackets(npackets * (ncbufs + 1));
685 #endif /* AFS_NT40_ENV */
687 /* Called from rxi_FindPeer, when initializing a clear rx_peer structure,
688 * to get interesting information.
689 * Curiously enough, the rx_peerHashTable_lock currently protects the
690 * Inited variable (and hence rx_GetIFInfo). When the fs suite uses
691 * pthreads, this issue will need to be revisited.
695 rxi_InitPeerParams(struct rx_peer *pp)
700 #if defined(ADAPT_PMTU) && defined(IP_MTU)
702 struct sockaddr_in addr;
709 * there's a race here since more than one thread could call
710 * rx_GetIFInfo. The race stops in rx_GetIFInfo.
718 /* try to second-guess IP, and identify which link is most likely to
719 * be used for traffic to/from this host. */
720 ppaddr = ntohl(pp->host);
724 pp->rateFlag = 2; /* start timing after two full packets */
725 /* I don't initialize these, because I presume they are bzero'd...
726 * pp->burstSize pp->burst pp->burstWait.sec pp->burstWait.usec
727 * pp->timeout.usec */
730 for (ix = 0; ix < rxi_numNetAddrs; ++ix) {
731 if ((rxi_NetAddrs[ix] & myNetMasks[ix]) == (ppaddr & myNetMasks[ix])) {
732 #ifdef IFF_POINTOPOINT
733 if (myNetFlags[ix] & IFF_POINTOPOINT)
735 #endif /* IFF_POINTOPOINT */
736 rxmtu = myNetMTUs[ix] - RX_IPUDP_SIZE;
737 if (rxmtu < RX_MIN_PACKET_SIZE)
738 rxmtu = RX_MIN_PACKET_SIZE;
739 if (pp->ifMTU < rxmtu)
740 pp->ifMTU = MIN(rx_MyMaxSendSize, rxmtu);
744 if (!pp->ifMTU) { /* not local */
746 pp->ifMTU = MIN(rx_MyMaxSendSize, RX_REMOTE_PACKET_SIZE);
748 #else /* ADAPT_MTU */
749 pp->rateFlag = 2; /* start timing after two full packets */
751 pp->ifMTU = MIN(rx_MyMaxSendSize, OLD_MAX_PACKET_SIZE);
752 #endif /* ADAPT_MTU */
753 #if defined(ADAPT_PMTU) && defined(IP_MTU)
754 sock=socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
756 addr.sin_family = AF_INET;
757 addr.sin_addr.s_addr = pp->host;
758 addr.sin_port = pp->port;
759 if (connect(sock, (struct sockaddr *)&addr, sizeof(addr)) == 0) {
761 socklen_t s = sizeof(mtu);
762 if (getsockopt(sock, SOL_IP, IP_MTU, &mtu, &s)== 0) {
763 pp->ifMTU = MIN(mtu - RX_IPUDP_SIZE, pp->ifMTU);
769 pp->ifMTU = rxi_AdjustIfMTU(pp->ifMTU);
770 pp->maxMTU = OLD_MAX_PACKET_SIZE; /* for compatibility with old guys */
771 pp->natMTU = MIN((int)pp->ifMTU, OLD_MAX_PACKET_SIZE);
772 pp->maxDgramPackets =
773 MIN(rxi_nDgramPackets,
774 rxi_AdjustDgramPackets(rxi_nSendFrags, pp->ifMTU));
776 MIN(rxi_nDgramPackets,
777 rxi_AdjustDgramPackets(rxi_nSendFrags, pp->ifMTU));
778 pp->maxDgramPackets = 1;
779 /* Initialize slow start parameters */
780 pp->MTU = MIN(pp->natMTU, pp->maxMTU);
782 pp->nDgramPackets = 1;
786 /* Don't expose jumobgram internals. */
790 rx_maxReceiveSize = OLD_MAX_PACKET_SIZE;
791 rxi_nSendFrags = rxi_nRecvFrags = 1;
794 /* Override max MTU. If rx_SetNoJumbo is called, it must be
795 called before calling rx_SetMaxMTU since SetNoJumbo clobbers rx_maxReceiveSize */
797 rx_SetMaxMTU(int mtu)
799 rx_MyMaxSendSize = rx_maxReceiveSizeUser = rx_maxReceiveSize = mtu;
802 #if defined(HAVE_LINUX_ERRQUEUE_H) && defined(ADAPT_PMTU)
804 rxi_HandleSocketError(int socket)
807 struct cmsghdr *cmsg;
808 struct sock_extended_err *err;
809 struct sockaddr_in addr;
810 struct sockaddr *offender;
811 char controlmsgbuf[256];
815 msg.msg_name = &addr;
816 msg.msg_namelen = sizeof(addr);
819 msg.msg_control = controlmsgbuf;
820 msg.msg_controllen = 256;
822 code = recvmsg(socket, &msg, MSG_ERRQUEUE|MSG_DONTWAIT|MSG_TRUNC);
824 if (code < 0 || !(msg.msg_flags & MSG_ERRQUEUE))
827 for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
828 if ((char *)cmsg - controlmsgbuf > msg.msg_controllen - CMSG_SPACE(0) ||
829 (char *)cmsg - controlmsgbuf > msg.msg_controllen - CMSG_SPACE(cmsg->cmsg_len) ||
830 cmsg->cmsg_len == 0) {
834 if (cmsg->cmsg_level == SOL_IP && cmsg->cmsg_type == IP_RECVERR)
840 err =(struct sock_extended_err *) CMSG_DATA(cmsg);
842 if (err->ee_errno == EMSGSIZE && err->ee_info >= 68) {
843 rxi_SetPeerMtu(addr.sin_addr.s_addr, addr.sin_port,
844 err->ee_info - RX_IPUDP_SIZE);
846 /* other DEST_UNREACH's and TIME_EXCEEDED should be dealt with too */