2 * Copyright 2000, International Business Machines Corporation and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
10 #include <afsconfig.h>
11 #include <afs/param.h>
16 #ifdef AFS_PTHREAD_ENV
17 # include <opr/lock.h>
20 #ifdef IGNORE_SOME_GCC_WARNINGS
21 # pragma GCC diagnostic warning "-Wstrict-prototypes"
24 #include <afs/pthread_glock.h>
28 #include <afs/rxgen_consts.h>
29 #define UBIK_LEGACY_CALLITER
32 short ubik_initializationState; /*!< initial state is zero */
36 * \brief Parse list for clients.
39 ubik_ParseClientList(int argc, char **argv, afs_uint32 * aothers)
48 inServer = 0; /* haven't seen -servers yet */
50 for (i = 1; i < argc; i++) {
51 /* look for -servers argument */
57 /* otherwise this is a new host name */
59 th = gethostbyname(tp);
64 memmove((void *)&temp, (const void *)th->h_addr,
67 if (counter++ >= MAXSERVERS)
71 /* haven't seen a -server yet */
72 if (!strcmp(tp, "-servers")) {
78 /* never saw a -server */
81 if (counter < MAXSERVERS)
82 *aothers++ = 0; /* null terminate if room */
86 #ifdef AFS_PTHREAD_ENV
89 static pthread_once_t random_once = PTHREAD_ONCE_INIT;
90 static int called_afs_random_once;
91 static pthread_key_t random_number_key;
96 opr_Verify(pthread_key_create(&random_number_key, NULL) == 0);
97 called_afs_random_once = 1;
103 * \brief use time and pid to try to get some initial randomness.
105 #define ranstage(x) (x)= (afs_uint32) (3141592621U*((afs_uint32)x)+1)
108 * \brief Random number generator and constants from KnuthV2 2d ed, p170
111 * X = (aX + c) % m \n
112 * m is a power of two \n
114 * a is 0.73m should be 0.01m .. 0.99m \n
115 * c is more or less immaterial. 1 or a is suggested. \n
117 * NB: LOW ORDER BITS are not very random. To get small random numbers,
118 * treat result as <1, with implied binary point, and multiply by
121 * NB: Has to be unsigned, since shifts on signed quantities may preserve
124 * In this case, m == 2^32, the mod operation is implicit. a == pi, which
125 * is used because it has some interesting characteristics (lacks any
126 * interesting bit-patterns).
131 #ifdef AFS_PTHREAD_ENV
134 if (!called_afs_random_once)
135 pthread_once(&random_once, afs_random_once);
137 state = (uintptr_t) pthread_getspecific(random_number_key);
139 static afs_uint32 state = 0;
144 state = time(0) + getpid();
145 for (i = 0; i < 15; i++) {
151 #ifdef AFS_PTHREAD_ENV
152 pthread_setspecific(random_number_key, (const void *)(uintptr_t)state);
159 * \brief Returns int 0..14 using the high bits of a pseudo-random number instead of
160 * the low bits, as the low bits are "less random" than the high ones...
162 * \todo Slight roundoff error exists, an excercise for the reader.
164 * Need to multiply by something with lots of ones in it, so multiply by
165 * 8 or 16 is right out.
168 afs_randomMod15(void)
172 temp = afs_random() >> 4;
173 temp = (temp * 15) >> 28;
181 #define abs(a) ((a) < 0 ? -1*(a) : (a))
183 ubik_ClientInit(struct rx_connection **serverconns,
184 struct ubik_client **aclient)
189 struct ubik_client *tc;
191 initialize_U_error_table();
193 if (*aclient) { /* the application is doing a re-initialization */
194 LOCK_UBIK_CLIENT((*aclient));
195 /* this is an important defensive check */
196 if (!((*aclient)->initializationState)) {
197 UNLOCK_UBIK_CLIENT((*aclient));
198 return UREINITIALIZE;
201 /* release all existing connections */
202 for (tc = *aclient, i = 0; i < MAXSERVERS; i++) {
203 struct rx_connection *rxConn = ubik_GetRPCConn(tc, i);
206 #ifdef AFS_PTHREAD_ENV
207 rx_ReleaseCachedConnection(rxConn);
209 rx_DestroyConnection(rxConn);
212 UNLOCK_UBIK_CLIENT((*aclient));
213 #ifdef AFS_PTHREAD_ENV
214 if (pthread_mutex_destroy(&((*aclient)->cm)))
215 return UMUTEXDESTROY;
218 tc = malloc(sizeof(struct ubik_client));
222 memset((void *)tc, 0, sizeof(*tc));
223 #ifdef AFS_PTHREAD_ENV
224 if (pthread_mutex_init(&(tc->cm), (const pthread_mutexattr_t *)0)) {
229 tc->initializationState = ++ubik_initializationState;
231 /* first count the # of server conns so we can randomize properly */
233 for (i = 0; i < MAXSERVERS; i++) {
234 if (serverconns[i] == (struct rx_connection *)0)
239 /* here count is the # of servers we're actually passed in. Compute
240 * offset, a number between 0..count-1, where we'll start copying from the
241 * client-provided array. */
242 for (i = 0; i < count; i++) {
243 offset = afs_randomMod15() % count;
244 for (j = abs(offset); j < 2 * count; j++) {
245 if (!tc->conns[abs(j % count)]) {
246 tc->conns[abs(j % count)] = serverconns[i];
257 * \brief Destroy an ubik connection.
259 * It calls rx to destroy the component rx connections, then frees the ubik
260 * connection structure.
263 ubik_ClientDestroy(struct ubik_client * aclient)
269 LOCK_UBIK_CLIENT(aclient);
270 for (c = 0; c < MAXSERVERS; c++) {
271 struct rx_connection *rxConn = ubik_GetRPCConn(aclient, c);
274 #ifdef AFS_PTHREAD_ENV
275 rx_ReleaseCachedConnection(rxConn);
277 rx_DestroyConnection(rxConn);
280 aclient->initializationState = 0; /* client in not initialized */
281 UNLOCK_UBIK_CLIENT(aclient);
282 #ifdef AFS_PTHREAD_ENV
283 pthread_mutex_destroy(&(aclient->cm)); /* ignore failure */
290 * \brief So that intermittent failures that cause connections to die
291 * don't kill whole ubik connection, refresh them when the connection is in
294 struct rx_connection *
295 ubik_RefreshConn(struct rx_connection *tc)
300 struct rx_securityClass *sc;
302 struct rx_connection *newTc;
304 host = rx_HostOf(rx_PeerOf(tc));
305 port = rx_PortOf(rx_PeerOf(tc));
306 service = rx_ServiceIdOf(tc);
307 sc = rx_SecurityObjectOf(tc);
308 si = rx_SecurityClassOf(tc);
311 * destroy old one after creating new one so that refCount on security
312 * object cannot reach zero.
314 newTc = rx_NewConnection(host, port, service, sc, si);
315 rx_DestroyConnection(tc);
319 #ifdef AFS_PTHREAD_ENV
321 pthread_once_t ubik_client_once = PTHREAD_ONCE_INIT;
322 pthread_mutex_t ubik_client_mutex;
323 #define LOCK_UCLNT_CACHE do { \
324 opr_Verify(pthread_once(&ubik_client_once, ubik_client_init_mutex) == 0); \
325 MUTEX_ENTER(&ubik_client_mutex); \
327 #define UNLOCK_UCLNT_CACHE MUTEX_EXIT(&ubik_client_mutex)
330 ubik_client_init_mutex(void)
332 MUTEX_INIT(&ubik_client_mutex, "client init", MUTEX_DEFAULT, 0);
337 #define LOCK_UCLNT_CACHE
338 #define UNLOCK_UCLNT_CACHE
343 static int *calls_needsync[SYNCCOUNT]; /* proc calls that need the sync site */
344 static int synccount = 0;
349 * \brief Call this after getting back a #UNOTSYNC.
351 * \note Getting a #UNOTSYNC error code back does \b not guarantee
352 * that there is a sync site yet elected. However, if there is a sync
353 * site out there somewhere, and you're trying an operation that
354 * requires a sync site, ubik will return #UNOTSYNC, indicating the
355 * operation won't work until you find a sync site
358 try_GetSyncSite(struct ubik_client *aclient, afs_int32 apos)
363 afs_int32 thisHost, newHost;
364 struct rx_connection *tc;
367 origLevel = aclient->initializationState;
370 tc = aclient->conns[apos];
371 if (tc && rx_ConnError(tc)) {
372 aclient->conns[apos] = (tc = ubik_RefreshConn(tc));
378 /* now see if we can find the sync site host */
379 code = VOTE_GetSyncSite(tc, &newHost);
380 if (aclient->initializationState != origLevel) {
381 return -1; /* somebody did a ubik_ClientInit */
384 if (!code && newHost) {
385 newHost = htonl(newHost); /* convert back to network order */
388 * position count at the appropriate slot in the client
389 * structure and retry. If we can't find in slot, we'll just
390 * continue through the whole list
392 for (i = 0; i < MAXSERVERS; i++) {
393 rxp = rx_PeerOf(aclient->conns[i]);
394 thisHost = rx_HostOf(rxp);
397 } else if (thisHost == newHost) {
398 return i; /* we were told to use this one */
409 * \brief Create an internal version of ubik_CallIter that takes an additional
410 * parameter - to indicate whether the ubik client handle has already
414 CallIter(int (*aproc) (), struct ubik_client *aclient,
415 afs_int32 aflags, int *apos, long p1, long p2, long p3, long p4,
416 long p5, long p6, long p7, long p8, long p9, long p10, long p11,
417 long p12, long p13, long p14, long p15, long p16, int needlock)
420 struct rx_connection *tc;
424 LOCK_UBIK_CLIENT(aclient);
426 origLevel = aclient->initializationState;
430 while (*apos < MAXSERVERS) {
431 /* tc is the next conn to try */
432 tc = aclient->conns[*apos];
436 if (rx_ConnError(tc)) {
437 tc = ubik_RefreshConn(tc);
438 aclient->conns[*apos] = tc;
441 if ((aflags & UPUBIKONLY) && (aclient->states[*apos] & CFLastFailed)) {
442 (*apos)++; /* try another one if this server is down */
444 break; /* this is the desired path */
447 if (*apos >= MAXSERVERS)
451 (*aproc) (tc, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13,
453 if (aclient->initializationState != origLevel)
454 /* somebody did a ubik_ClientInit */
457 /* what should I do in case of UNOQUORUM ? */
459 aclient->states[*apos] |= CFLastFailed; /* network errors */
461 /* either misc ubik code, or misc application code or success. */
462 aclient->states[*apos] &= ~CFLastFailed; /* operation worked */
468 UNLOCK_UBIK_CLIENT(aclient);
474 * \brief This is part of an iterator. It doesn't handle finding sync sites.
477 ubik_CallIter(int (*aproc) (), struct ubik_client *aclient,
478 afs_int32 aflags, int *apos, long p1, long p2,
479 long p3, long p4, long p5, long p6, long p7,
480 long p8, long p9, long p10, long p11, long p12,
481 long p13, long p14, long p15, long p16)
483 return CallIter(aproc, aclient, aflags, apos, p1, p2, p3, p4, p5, p6, p7,
484 p8, p9, p10, p11, p12, p13, p14, p15, p16, NEED_LOCK);
488 * \brief Call this instead of stub and we'll guarantee to find a host that's up.
490 * \todo In the future, we should also put in a protocol to find the sync site.
493 ubik_Call_New(int (*aproc) (), struct ubik_client *aclient,
494 afs_int32 aflags, long p1, long p2, long p3, long p4, long p5,
495 long p6, long p7, long p8, long p9, long p10, long p11,
496 long p12, long p13, long p14, long p15, long p16)
498 afs_int32 code, rcode;
505 LOCK_UBIK_CLIENT(aclient);
508 origLevel = aclient->initializationState;
510 /* Do two passes. First pass only checks servers known running */
511 for (aflags |= UPUBIKONLY, pass = 0; pass < 2;
512 pass++, aflags &= ~UPUBIKONLY) {
517 CallIter(aproc, aclient, aflags, &count, p1, p2, p3, p4, p5,
518 p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16,
520 if (code && (aclient->initializationState != origLevel)) {
523 if (code == UNOSERVERS) {
526 rcode = code; /* remember code from last good call */
528 if (code == UNOTSYNC) { /* means this requires a sync site */
529 if (aclient->conns[3]) { /* don't bother unless 4 or more srv */
530 temp = try_GetSyncSite(aclient, count);
531 if (aclient->initializationState != origLevel) {
532 goto restart; /* somebody did a ubik_ClientInit */
534 if ((temp >= 0) && ((temp > count) || (stepBack++ <= 2))) {
535 count = temp; /* generally try to make progress */
538 } else if ((code >= 0) && (code != UNOQUORUM)) {
539 UNLOCK_UBIK_CLIENT(aclient);
540 return code; /* success or global error condition */
544 UNLOCK_UBIK_CLIENT(aclient);
549 * call this instead of stub and we'll guarantee to find a host that's up.
551 * \todo In the future, we should also put in a protocol to find the sync site.
554 ubik_Call(int (*aproc) (), struct ubik_client *aclient,
555 afs_int32 aflags, long p1, long p2, long p3, long p4,
556 long p5, long p6, long p7, long p8, long p9, long p10,
557 long p11, long p12, long p13, long p14, long p15, long p16)
559 afs_int32 rcode, code, newHost, thisHost, i, count;
560 int chaseCount, pass, needsync, inlist, j;
561 struct rx_connection *tc;
565 if (aflags & UBIK_CALL_NEW)
566 return ubik_Call_New(aproc, aclient, aflags, p1, p2, p3, p4,
567 p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15,
572 LOCK_UBIK_CLIENT(aclient);
575 origLevel = aclient->initializationState;
577 chaseCount = inlist = needsync = 0;
580 for (j = 0; ((j < SYNCCOUNT) && calls_needsync[j]); j++) {
581 if (calls_needsync[j] == (int *)aproc) {
582 inlist = needsync = 1;
588 * First pass, we try all servers that are up.
589 * Second pass, we try all servers.
591 for (pass = 0; pass < 2; pass++) { /*p */
592 /* For each entry in our servers list */
593 for (count = 0;; count++) { /*s */
596 /* Need a sync site. Lets try to quickly find it */
597 if (aclient->syncSite) {
598 newHost = aclient->syncSite; /* already in network order */
599 aclient->syncSite = 0; /* Will reset if it works */
600 } else if (aclient->conns[3]) {
601 /* If there are fewer than four db servers in a cell,
602 * there's no point in making the GetSyncSite call.
603 * At best, it's a wash. At worst, it results in more
604 * RPCs than you would otherwise make.
606 tc = aclient->conns[count];
607 if (tc && rx_ConnError(tc)) {
608 aclient->conns[count] = tc = ubik_RefreshConn(tc);
612 code = VOTE_GetSyncSite(tc, &newHost);
613 if (aclient->initializationState != origLevel)
614 goto restart; /* somebody did a ubik_ClientInit */
617 newHost = htonl(newHost); /* convert to network order */
622 /* position count at the appropriate slot in the client
623 * structure and retry. If we can't find in slot, we'll
624 * just continue through the whole list
626 for (i = 0; i < MAXSERVERS && aclient->conns[i]; i++) {
627 rxp = rx_PeerOf(aclient->conns[i]);
628 thisHost = rx_HostOf(rxp);
631 if (thisHost == newHost) {
632 if (chaseCount++ > 2)
633 break; /* avoid loop asking */
634 count = i; /* this index is the sync site */
641 tc = aclient->conns[count];
642 if (tc && rx_ConnError(tc)) {
643 aclient->conns[count] = tc = ubik_RefreshConn(tc);
648 if ((pass == 0) && (aclient->states[count] & CFLastFailed)) {
649 continue; /* this guy's down */
653 (*aproc) (tc, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11,
654 p12, p13, p14, p15, p16);
655 if (aclient->initializationState != origLevel) {
656 /* somebody did a ubik_ClientInit */
658 goto restart; /* call failed */
660 goto done; /* call suceeded */
662 if (rcode < 0) { /* network errors */
663 aclient->states[count] |= CFLastFailed; /* Mark serer down */
664 } else if (rcode == UNOTSYNC) {
666 } else if (rcode != UNOQUORUM) {
667 /* either misc ubik code, or misc appl code, or success. */
668 aclient->states[count] &= ~CFLastFailed; /* mark server up */
669 goto done; /* all done */
676 if (!inlist) { /* Remember proc call that needs sync site */
678 calls_needsync[synccount % SYNCCOUNT] = (int *)aproc;
683 if (!rcode) { /* Remember the sync site - cmd successful */
684 rxp = rx_PeerOf(aclient->conns[count]);
685 aclient->syncSite = rx_HostOf(rxp);
688 UNLOCK_UBIK_CLIENT(aclient);