2 * Copyright 2000, International Business Machines Corporation and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
10 #include <afsconfig.h>
11 #include <afs/param.h>
15 #ifdef IGNORE_SOME_GCC_WARNINGS
16 # pragma GCC diagnostic warning "-Wstrict-prototypes"
20 #include "afsincludes.h"
24 #include <afs/pthread_glock.h>
35 #include <netinet/in.h>
37 #include <afs/rxgen_consts.h>
38 #define UBIK_LEGACY_CALLITER
45 short ubik_initializationState; /*!< initial state is zero */
49 * \brief Parse list for clients.
52 ubik_ParseClientList(int argc, char **argv, afs_uint32 * aothers)
61 inServer = 0; /* haven't seen -servers yet */
63 for (i = 1; i < argc; i++) {
64 /* look for -servers argument */
70 /* otherwise this is a new host name */
72 th = gethostbyname(tp);
77 memmove((void *)&temp, (const void *)th->h_addr,
80 if (counter++ >= MAXSERVERS)
84 /* haven't seen a -server yet */
85 if (!strcmp(tp, "-servers")) {
91 /* never saw a -server */
94 if (counter < MAXSERVERS)
95 *aothers++ = 0; /* null terminate if room */
99 #include <afs/afs_assert.h>
100 #ifdef AFS_PTHREAD_ENV
103 static pthread_once_t random_once = PTHREAD_ONCE_INIT;
104 static int called_afs_random_once;
105 static pthread_key_t random_number_key;
108 afs_random_once(void)
110 osi_Assert(pthread_key_create(&random_number_key, NULL) == 0);
111 called_afs_random_once = 1;
116 #if !defined(UKERNEL)
118 * \brief use time and pid to try to get some initial randomness.
120 #define ranstage(x) (x)= (afs_uint32) (3141592621U*((afs_uint32)x)+1)
123 * \brief Random number generator and constants from KnuthV2 2d ed, p170
126 * X = (aX + c) % m \n
127 * m is a power of two \n
129 * a is 0.73m should be 0.01m .. 0.99m \n
130 * c is more or less immaterial. 1 or a is suggested. \n
132 * NB: LOW ORDER BITS are not very random. To get small random numbers,
133 * treat result as <1, with implied binary point, and multiply by
136 * NB: Has to be unsigned, since shifts on signed quantities may preserve
139 * In this case, m == 2^32, the mod operation is implicit. a == pi, which
140 * is used because it has some interesting characteristics (lacks any
141 * interesting bit-patterns).
146 #ifdef AFS_PTHREAD_ENV
149 if (!called_afs_random_once)
150 pthread_once(&random_once, afs_random_once);
152 state = (uintptr_t) pthread_getspecific(random_number_key);
154 static afs_uint32 state = 0;
159 state = time(0) + getpid();
160 for (i = 0; i < 15; i++) {
166 #ifdef AFS_PTHREAD_ENV
167 pthread_setspecific(random_number_key, (const void *)(uintptr_t)state);
174 * \brief Returns int 0..14 using the high bits of a pseudo-random number instead of
175 * the low bits, as the low bits are "less random" than the high ones...
177 * \todo Slight roundoff error exists, an excercise for the reader.
179 * Need to multiply by something with lots of ones in it, so multiply by
180 * 8 or 16 is right out.
183 afs_randomMod15(void)
187 temp = afs_random() >> 4;
188 temp = (temp * 15) >> 28;
192 #endif /* !defined(UKERNEL) */
197 #define abs(a) ((a) < 0 ? -1*(a) : (a))
199 ubik_ClientInit(struct rx_connection **serverconns,
200 struct ubik_client **aclient)
205 struct ubik_client *tc;
207 initialize_U_error_table();
209 if (*aclient) { /* the application is doing a re-initialization */
210 LOCK_UBIK_CLIENT((*aclient));
211 /* this is an important defensive check */
212 if (!((*aclient)->initializationState)) {
213 UNLOCK_UBIK_CLIENT((*aclient));
214 return UREINITIALIZE;
217 /* release all existing connections */
218 for (tc = *aclient, i = 0; i < MAXSERVERS; i++) {
219 struct rx_connection *rxConn = ubik_GetRPCConn(tc, i);
222 #ifdef AFS_PTHREAD_ENV
223 rx_ReleaseCachedConnection(rxConn);
225 rx_DestroyConnection(rxConn);
228 UNLOCK_UBIK_CLIENT((*aclient));
229 #ifdef AFS_PTHREAD_ENV
230 if (pthread_mutex_destroy(&((*aclient)->cm)))
231 return UMUTEXDESTROY;
234 tc = (struct ubik_client *)malloc(sizeof(struct ubik_client));
238 memset((void *)tc, 0, sizeof(*tc));
239 #ifdef AFS_PTHREAD_ENV
240 if (pthread_mutex_init(&(tc->cm), (const pthread_mutexattr_t *)0)) {
244 tc->initializationState = ++ubik_initializationState;
246 /* first count the # of server conns so we can randomize properly */
248 for (i = 0; i < MAXSERVERS; i++) {
249 if (serverconns[i] == (struct rx_connection *)0)
254 /* here count is the # of servers we're actually passed in. Compute
255 * offset, a number between 0..count-1, where we'll start copying from the
256 * client-provided array. */
257 for (i = 0; i < count; i++) {
258 offset = afs_randomMod15() % count;
259 for (j = abs(offset); j < 2 * count; j++) {
260 if (!tc->conns[abs(j % count)]) {
261 tc->conns[abs(j % count)] = serverconns[i];
272 * \brief Destroy an ubik connection.
274 * It calls rx to destroy the component rx connections, then frees the ubik
275 * connection structure.
278 ubik_ClientDestroy(struct ubik_client * aclient)
284 LOCK_UBIK_CLIENT(aclient);
285 for (c = 0; c < MAXSERVERS; c++) {
286 struct rx_connection *rxConn = ubik_GetRPCConn(aclient, c);
289 #ifdef AFS_PTHREAD_ENV
290 rx_ReleaseCachedConnection(rxConn);
292 rx_DestroyConnection(rxConn);
295 aclient->initializationState = 0; /* client in not initialized */
296 UNLOCK_UBIK_CLIENT(aclient);
297 #ifdef AFS_PTHREAD_ENV
298 pthread_mutex_destroy(&(aclient->cm)); /* ignore failure */
305 * \brief So that intermittent failures that cause connections to die
306 * don't kill whole ubik connection, refresh them when the connection is in
309 struct rx_connection *
310 ubik_RefreshConn(struct rx_connection *tc)
315 struct rx_securityClass *sc;
317 struct rx_connection *newTc;
319 host = rx_HostOf(rx_PeerOf(tc));
320 port = rx_PortOf(rx_PeerOf(tc));
321 service = rx_ServiceIdOf(tc);
322 sc = rx_SecurityObjectOf(tc);
323 si = rx_SecurityClassOf(tc);
326 * destroy old one after creating new one so that refCount on security
327 * object cannot reach zero.
329 newTc = rx_NewConnection(host, port, service, sc, si);
330 rx_DestroyConnection(tc);
334 #ifdef AFS_PTHREAD_ENV
336 pthread_once_t ubik_client_once = PTHREAD_ONCE_INIT;
337 pthread_mutex_t ubik_client_mutex;
338 #define LOCK_UCLNT_CACHE do { \
339 osi_Assert(pthread_once(&ubik_client_once, ubik_client_init_mutex) == 0); \
340 MUTEX_ENTER(&ubik_client_mutex); \
342 #define UNLOCK_UCLNT_CACHE MUTEX_EXIT(&ubik_client_mutex)
345 ubik_client_init_mutex(void)
347 MUTEX_INIT(&ubik_client_mutex, "client init", MUTEX_DEFAULT, 0);
352 #define LOCK_UCLNT_CACHE
353 #define UNLOCK_UCLNT_CACHE
358 static int *calls_needsync[SYNCCOUNT]; /* proc calls that need the sync site */
359 static int synccount = 0;
364 * \brief Call this after getting back a #UNOTSYNC.
366 * \note Getting a #UNOTSYNC error code back does \b not guarantee
367 * that there is a sync site yet elected. However, if there is a sync
368 * site out there somewhere, and you're trying an operation that
369 * requires a sync site, ubik will return #UNOTSYNC, indicating the
370 * operation won't work until you find a sync site
373 try_GetSyncSite(struct ubik_client *aclient, afs_int32 apos)
378 afs_int32 thisHost, newHost;
379 struct rx_connection *tc;
382 origLevel = aclient->initializationState;
385 tc = aclient->conns[apos];
386 if (tc && rx_ConnError(tc)) {
387 aclient->conns[apos] = (tc = ubik_RefreshConn(tc));
393 /* now see if we can find the sync site host */
394 code = VOTE_GetSyncSite(tc, &newHost);
395 if (aclient->initializationState != origLevel) {
396 return -1; /* somebody did a ubik_ClientInit */
399 if (!code && newHost) {
400 newHost = htonl(newHost); /* convert back to network order */
403 * position count at the appropriate slot in the client
404 * structure and retry. If we can't find in slot, we'll just
405 * continue through the whole list
407 for (i = 0; i < MAXSERVERS; i++) {
408 rxp = rx_PeerOf(aclient->conns[i]);
409 thisHost = rx_HostOf(rxp);
412 } else if (thisHost == newHost) {
413 return i; /* we were told to use this one */
424 * \brief Create an internal version of ubik_CallIter that takes an additional
425 * parameter - to indicate whether the ubik client handle has already
429 CallIter(int (*aproc) (), struct ubik_client *aclient,
430 afs_int32 aflags, int *apos, long p1, long p2, long p3, long p4,
431 long p5, long p6, long p7, long p8, long p9, long p10, long p11,
432 long p12, long p13, long p14, long p15, long p16, int needlock)
435 struct rx_connection *tc;
439 LOCK_UBIK_CLIENT(aclient);
441 origLevel = aclient->initializationState;
445 while (*apos < MAXSERVERS) {
446 /* tc is the next conn to try */
447 tc = aclient->conns[*apos];
450 UNLOCK_UBIK_CLIENT(aclient);
455 if (rx_ConnError(tc)) {
456 tc = ubik_RefreshConn(tc);
457 aclient->conns[*apos] = tc;
460 if ((aflags & UPUBIKONLY) && (aclient->states[*apos] & CFLastFailed)) {
461 (*apos)++; /* try another one if this server is down */
463 break; /* this is the desired path */
466 if (*apos >= MAXSERVERS) {
468 UNLOCK_UBIK_CLIENT(aclient);
474 (*aproc) (tc, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13,
476 if (aclient->initializationState != origLevel) {
478 UNLOCK_UBIK_CLIENT(aclient);
480 return code; /* somebody did a ubik_ClientInit */
483 /* what should I do in case of UNOQUORUM ? */
485 aclient->states[*apos] |= CFLastFailed; /* network errors */
487 /* either misc ubik code, or misc application code or success. */
488 aclient->states[*apos] &= ~CFLastFailed; /* operation worked */
493 UNLOCK_UBIK_CLIENT(aclient);
499 * \brief This is part of an iterator. It doesn't handle finding sync sites.
502 ubik_CallIter(int (*aproc) (), struct ubik_client *aclient,
503 afs_int32 aflags, int *apos, long p1, long p2,
504 long p3, long p4, long p5, long p6, long p7,
505 long p8, long p9, long p10, long p11, long p12,
506 long p13, long p14, long p15, long p16)
508 return CallIter(aproc, aclient, aflags, apos, p1, p2, p3, p4, p5, p6, p7,
509 p8, p9, p10, p11, p12, p13, p14, p15, p16, NEED_LOCK);
513 * \brief Call this instead of stub and we'll guarantee to find a host that's up.
515 * \todo In the future, we should also put in a protocol to find the sync site.
518 ubik_Call_New(int (*aproc) (), struct ubik_client *aclient,
519 afs_int32 aflags, long p1, long p2, long p3, long p4, long p5,
520 long p6, long p7, long p8, long p9, long p10, long p11,
521 long p12, long p13, long p14, long p15, long p16)
523 afs_int32 code, rcode;
530 LOCK_UBIK_CLIENT(aclient);
533 origLevel = aclient->initializationState;
535 /* Do two passes. First pass only checks servers known running */
536 for (aflags |= UPUBIKONLY, pass = 0; pass < 2;
537 pass++, aflags &= ~UPUBIKONLY) {
542 CallIter(aproc, aclient, aflags, &count, p1, p2, p3, p4, p5,
543 p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16,
545 if (code && (aclient->initializationState != origLevel)) {
548 if (code == UNOSERVERS) {
551 rcode = code; /* remember code from last good call */
553 if (code == UNOTSYNC) { /* means this requires a sync site */
554 if (aclient->conns[3]) { /* don't bother unless 4 or more srv */
555 temp = try_GetSyncSite(aclient, count);
556 if (aclient->initializationState != origLevel) {
557 goto restart; /* somebody did a ubik_ClientInit */
559 if ((temp >= 0) && ((temp > count) || (stepBack++ <= 2))) {
560 count = temp; /* generally try to make progress */
563 } else if ((code >= 0) && (code != UNOQUORUM)) {
564 UNLOCK_UBIK_CLIENT(aclient);
565 return code; /* success or global error condition */
569 UNLOCK_UBIK_CLIENT(aclient);
574 * call this instead of stub and we'll guarantee to find a host that's up.
576 * \todo In the future, we should also put in a protocol to find the sync site.
579 ubik_Call(int (*aproc) (), struct ubik_client *aclient,
580 afs_int32 aflags, long p1, long p2, long p3, long p4,
581 long p5, long p6, long p7, long p8, long p9, long p10,
582 long p11, long p12, long p13, long p14, long p15, long p16)
584 afs_int32 rcode, code, newHost, thisHost, i, count;
585 int chaseCount, pass, needsync, inlist, j;
586 struct rx_connection *tc;
590 if (aflags & UBIK_CALL_NEW)
591 return ubik_Call_New(aproc, aclient, aflags, p1, p2, p3, p4,
592 p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15,
597 LOCK_UBIK_CLIENT(aclient);
600 origLevel = aclient->initializationState;
602 chaseCount = inlist = needsync = 0;
605 for (j = 0; ((j < SYNCCOUNT) && calls_needsync[j]); j++) {
606 if (calls_needsync[j] == (int *)aproc) {
607 inlist = needsync = 1;
613 * First pass, we try all servers that are up.
614 * Second pass, we try all servers.
616 for (pass = 0; pass < 2; pass++) { /*p */
617 /* For each entry in our servers list */
618 for (count = 0;; count++) { /*s */
621 /* Need a sync site. Lets try to quickly find it */
622 if (aclient->syncSite) {
623 newHost = aclient->syncSite; /* already in network order */
624 aclient->syncSite = 0; /* Will reset if it works */
625 } else if (aclient->conns[3]) {
626 /* If there are fewer than four db servers in a cell,
627 * there's no point in making the GetSyncSite call.
628 * At best, it's a wash. At worst, it results in more
629 * RPCs than you would otherwise make.
631 tc = aclient->conns[count];
632 if (tc && rx_ConnError(tc)) {
633 aclient->conns[count] = tc = ubik_RefreshConn(tc);
637 code = VOTE_GetSyncSite(tc, &newHost);
638 if (aclient->initializationState != origLevel)
639 goto restart; /* somebody did a ubik_ClientInit */
642 newHost = htonl(newHost); /* convert to network order */
647 /* position count at the appropriate slot in the client
648 * structure and retry. If we can't find in slot, we'll
649 * just continue through the whole list
651 for (i = 0; i < MAXSERVERS && aclient->conns[i]; i++) {
652 rxp = rx_PeerOf(aclient->conns[i]);
653 thisHost = rx_HostOf(rxp);
656 if (thisHost == newHost) {
657 if (chaseCount++ > 2)
658 break; /* avoid loop asking */
659 count = i; /* this index is the sync site */
666 tc = aclient->conns[count];
667 if (tc && rx_ConnError(tc)) {
668 aclient->conns[count] = tc = ubik_RefreshConn(tc);
673 if ((pass == 0) && (aclient->states[count] & CFLastFailed)) {
674 continue; /* this guy's down */
678 (*aproc) (tc, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11,
679 p12, p13, p14, p15, p16);
680 if (aclient->initializationState != origLevel) {
681 /* somebody did a ubik_ClientInit */
683 goto restart; /* call failed */
685 goto done; /* call suceeded */
687 if (rcode < 0) { /* network errors */
688 aclient->states[count] |= CFLastFailed; /* Mark serer down */
689 } else if (rcode == UNOTSYNC) {
691 } else if (rcode != UNOQUORUM) {
692 /* either misc ubik code, or misc appl code, or success. */
693 aclient->states[count] &= ~CFLastFailed; /* mark server up */
694 goto done; /* all done */
701 if (!inlist) { /* Remember proc call that needs sync site */
703 calls_needsync[synccount % SYNCCOUNT] = (int *)aproc;
708 if (!rcode) { /* Remember the sync site - cmd successful */
709 rxp = rx_PeerOf(aclient->conns[count]);
710 aclient->syncSite = rx_HostOf(rxp);
713 UNLOCK_UBIK_CLIENT(aclient);