2 * Copyright 2000, International Business Machines Corporation and others.
5 * This software has been released under the terms of the IBM Public
6 * License. For details, see the LICENSE file in the top-level source
7 * directory or online at http://www.openafs.org/dl/license10.html
10 #include <afsconfig.h>
12 #include "afs/param.h"
14 #include <afs/param.h>
21 #include "afs/sysincludes.h"
22 #include "afsincludes.h"
27 #include "afs/rxgen_consts.h"
28 #define UBIK_LEGACY_CALLITER 1
30 #include "afs/pthread_glock.h"
31 #else /* defined(UKERNEL) */
33 #include <afs/pthread_glock.h>
44 #include <netinet/in.h>
46 #include <afs/rxgen_consts.h>
48 #endif /* defined(UKERNEL) */
51 short ubik_initializationState; /*!< initial state is zero */
55 * \brief Parse list for clients.
58 ubik_ParseClientList(int argc, char **argv, afs_int32 * aothers)
62 register struct hostent *th;
63 afs_int32 temp, counter;
66 inServer = 0; /* haven't seen -servers yet */
68 for (i = 1; i < argc; i++) {
69 /* look for -servers argument */
75 /* otherwise this is a new host name */
77 th = gethostbyname(tp);
82 memmove((void *)&temp, (const void *)th->h_addr,
85 if (counter++ >= MAXSERVERS)
89 /* haven't seen a -server yet */
90 if (!strcmp(tp, "-servers")) {
96 /* never saw a -server */
99 if (counter < MAXSERVERS)
100 *aothers++ = 0; /* null terminate if room */
104 #ifdef AFS_PTHREAD_ENV
108 static pthread_once_t random_once = PTHREAD_ONCE_INIT;
109 static int called_afs_random_once;
110 static pthread_key_t random_number_key;
113 afs_random_once(void)
115 assert(pthread_key_create(&random_number_key, NULL) == 0);
116 called_afs_random_once = 1;
121 #if !defined(UKERNEL)
123 * \brief use time and pid to try to get some initial randomness.
125 #define ranstage(x) (x)= (afs_uint32) (3141592621U*((afs_uint32)x)+1)
128 * \brief Random number generator and constants from KnuthV2 2d ed, p170
131 * X = (aX + c) % m \n
132 * m is a power of two \n
134 * a is 0.73m should be 0.01m .. 0.99m \n
135 * c is more or less immaterial. 1 or a is suggested. \n
137 * NB: LOW ORDER BITS are not very random. To get small random numbers,
138 * treat result as <1, with implied binary point, and multiply by
141 * NB: Has to be unsigned, since shifts on signed quantities may preserve
144 * In this case, m == 2^32, the mod operation is implicit. a == pi, which
145 * is used because it has some interesting characteristics (lacks any
146 * interesting bit-patterns).
151 #ifdef AFS_PTHREAD_ENV
154 (called_afs_random_once || pthread_once(&random_once, afs_random_once));
155 state = (afs_uint32) pthread_getspecific(random_number_key);
157 static afs_uint32 state = 0;
162 state = time(0) + getpid();
163 for (i = 0; i < 15; i++) {
169 #ifdef AFS_PTHREAD_ENV
170 pthread_setspecific(random_number_key, (const void *)state);
177 * \brief Returns int 0..14 using the high bits of a pseudo-random number instead of
178 * the low bits, as the low bits are "less random" than the high ones...
180 * \todo Slight roundoff error exists, an excercise for the reader.
182 * Need to multiply by something with lots of ones in it, so multiply by
183 * 8 or 16 is right out.
186 afs_randomMod15(void)
190 temp = afs_random() >> 4;
191 temp = (temp * 15) >> 28;
195 #endif /* !defined(UKERNEL) */
200 #define abs(a) ((a) < 0 ? -1*(a) : (a))
202 ubik_ClientInit(register struct rx_connection **serverconns,
203 struct ubik_client **aclient)
208 register struct ubik_client *tc;
210 initialize_U_error_table();
212 if (*aclient) { /* the application is doing a re-initialization */
213 LOCK_UBIK_CLIENT((*aclient));
214 /* this is an important defensive check */
215 if (!((*aclient)->initializationState)) {
216 UNLOCK_UBIK_CLIENT((*aclient));
217 return UREINITIALIZE;
220 /* release all existing connections */
221 for (tc = *aclient, i = 0; i < MAXSERVERS; i++) {
222 struct rx_connection *rxConn = ubik_GetRPCConn(tc, i);
225 #ifdef AFS_PTHREAD_ENV
226 rx_ReleaseCachedConnection(rxConn);
228 rx_DestroyConnection(rxConn);
231 UNLOCK_UBIK_CLIENT((*aclient));
232 #ifdef AFS_PTHREAD_ENV
233 if (pthread_mutex_destroy(&((*aclient)->cm)))
234 return UMUTEXDESTROY;
237 tc = (struct ubik_client *)malloc(sizeof(struct ubik_client));
241 memset((void *)tc, 0, sizeof(*tc));
242 #ifdef AFS_PTHREAD_ENV
243 if (pthread_mutex_init(&(tc->cm), (const pthread_mutexattr_t *)0)) {
247 tc->initializationState = ++ubik_initializationState;
249 /* first count the # of server conns so we can randomize properly */
251 for (i = 0; i < MAXSERVERS; i++) {
252 if (serverconns[i] == (struct rx_connection *)0)
257 /* here count is the # of servers we're actually passed in. Compute
258 * offset, a number between 0..count-1, where we'll start copying from the
259 * client-provided array. */
260 for (i = 0; i < count; i++) {
261 offset = afs_randomMod15() % count;
262 for (j = abs(offset); j < 2 * count; j++) {
263 if (!tc->conns[abs(j % count)]) {
264 tc->conns[abs(j % count)] = serverconns[i];
275 * \brief Destroy an ubik connection.
277 * It calls rx to destroy the component rx connections, then frees the ubik
278 * connection structure.
281 ubik_ClientDestroy(struct ubik_client * aclient)
287 LOCK_UBIK_CLIENT(aclient);
288 for (c = 0; c < MAXSERVERS; c++) {
289 struct rx_connection *rxConn = ubik_GetRPCConn(aclient, c);
292 #ifdef AFS_PTHREAD_ENV
293 rx_ReleaseCachedConnection(rxConn);
295 rx_DestroyConnection(rxConn);
298 aclient->initializationState = 0; /* client in not initialized */
299 UNLOCK_UBIK_CLIENT(aclient);
300 #ifdef AFS_PTHREAD_ENV
301 pthread_mutex_destroy(&(aclient->cm)); /* ignore failure */
308 * \brief So that intermittent failures that cause connections to die
309 * don't kill whole ubik connection, refresh them when the connection is in
312 struct rx_connection *
313 ubik_RefreshConn(struct rx_connection *tc)
318 struct rx_securityClass *sc;
320 struct rx_connection *newTc;
322 host = rx_HostOf(rx_PeerOf(tc));
323 port = rx_PortOf(rx_PeerOf(tc));
324 service = rx_ServiceIdOf(tc);
325 sc = rx_SecurityObjectOf(tc);
326 si = rx_SecurityClassOf(tc);
329 * destroy old one after creating new one so that refCount on security
330 * object cannot reach zero.
332 newTc = rx_NewConnection(host, port, service, sc, si);
333 rx_DestroyConnection(tc);
337 #ifdef AFS_PTHREAD_ENV
339 pthread_once_t ubik_client_once = PTHREAD_ONCE_INIT;
340 pthread_mutex_t ubik_client_mutex;
341 #define LOCK_UCLNT_CACHE \
342 assert(pthread_once(&ubik_client_once, ubik_client_init_mutex) == 0 && \
343 pthread_mutex_lock(&ubik_client_mutex)==0)
344 #define UNLOCK_UCLNT_CACHE assert(pthread_mutex_unlock(&ubik_client_mutex)==0)
347 ubik_client_init_mutex(void)
349 assert(pthread_mutex_init(&ubik_client_mutex, NULL) == 0);
354 #define LOCK_UCLNT_CACHE
355 #define UNLOCK_UCLNT_CACHE
360 static int *calls_needsync[SYNCCOUNT]; /* proc calls that need the sync site */
361 static int synccount = 0;
364 * call this instead of stub and we'll guarantee to find a host that's up.
366 * \todo In the future, we should also put in a protocol to find the sync site.
369 ubik_Call(int (*aproc) (), register struct ubik_client *aclient,
370 afs_int32 aflags, long p1, long p2, long p3, long p4,
371 long p5, long p6, long p7, long p8, long p9, long p10,
372 long p11, long p12, long p13, long p14, long p15, long p16)
374 afs_int32 rcode, code, newHost, thisHost, i, count;
375 int chaseCount, pass, needsync, inlist, j;
376 struct rx_connection *tc;
382 LOCK_UBIK_CLIENT(aclient);
385 origLevel = aclient->initializationState;
387 chaseCount = inlist = needsync = 0;
390 for (j = 0; ((j < SYNCCOUNT) && calls_needsync[j]); j++) {
391 if (calls_needsync[j] == (int *)aproc) {
392 inlist = needsync = 1;
398 * First pass, we try all servers that are up.
399 * Second pass, we try all servers.
401 for (pass = 0; pass < 2; pass++) { /*p */
402 /* For each entry in our servers list */
403 for (count = 0;; count++) { /*s */
406 /* Need a sync site. Lets try to quickly find it */
407 if (aclient->syncSite) {
408 newHost = aclient->syncSite; /* already in network order */
409 aclient->syncSite = 0; /* Will reset if it works */
410 } else if (aclient->conns[3]) {
411 /* If there are fewer than four db servers in a cell,
412 * there's no point in making the GetSyncSite call.
413 * At best, it's a wash. At worst, it results in more
414 * RPCs than you would otherwise make.
416 tc = aclient->conns[count];
417 if (tc && rx_ConnError(tc)) {
418 aclient->conns[count] = tc = ubik_RefreshConn(tc);
422 code = VOTE_GetSyncSite(tc, &newHost);
423 if (aclient->initializationState != origLevel)
424 goto restart; /* somebody did a ubik_ClientInit */
427 newHost = htonl(newHost); /* convert to network order */
432 /* position count at the appropriate slot in the client
433 * structure and retry. If we can't find in slot, we'll
434 * just continue through the whole list
436 for (i = 0; i < MAXSERVERS && aclient->conns[i]; i++) {
437 rxp = rx_PeerOf(aclient->conns[i]);
438 thisHost = rx_HostOf(rxp);
441 if (thisHost == newHost) {
442 if (chaseCount++ > 2)
443 break; /* avoid loop asking */
444 count = i; /* this index is the sync site */
451 tc = aclient->conns[count];
452 if (tc && rx_ConnError(tc)) {
453 aclient->conns[count] = tc = ubik_RefreshConn(tc);
458 if ((pass == 0) && (aclient->states[count] & CFLastFailed)) {
459 continue; /* this guy's down */
463 (*aproc) (tc, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11,
464 p12, p13, p14, p15, p16);
465 if (aclient->initializationState != origLevel) {
466 /* somebody did a ubik_ClientInit */
468 goto restart; /* call failed */
470 goto done; /* call suceeded */
472 if (rcode < 0) { /* network errors */
473 aclient->states[count] |= CFLastFailed; /* Mark serer down */
474 } else if (rcode == UNOTSYNC) {
476 } else if (rcode != UNOQUORUM) {
477 /* either misc ubik code, or misc appl code, or success. */
478 aclient->states[count] &= ~CFLastFailed; /* mark server up */
479 goto done; /* all done */
486 if (!inlist) { /* Remember proc call that needs sync site */
488 calls_needsync[synccount % SYNCCOUNT] = (int *)aproc;
493 if (!rcode) { /* Remember the sync site - cmd successful */
494 rxp = rx_PeerOf(aclient->conns[count]);
495 aclient->syncSite = rx_HostOf(rxp);
498 UNLOCK_UBIK_CLIENT(aclient);
505 * \brief Call this after getting back a #UNOTSYNC.
507 * \note Getting a #UNOTSYNC error code back does \b not guarantee
508 * that there is a sync site yet elected. However, if there is a sync
509 * site out there somewhere, and you're trying an operation that
510 * requires a sync site, ubik will return #UNOTSYNC, indicating the
511 * operation won't work until you find a sync site
514 try_GetSyncSite(register struct ubik_client *aclient, afs_int32 apos)
519 afs_int32 thisHost, newHost;
520 struct rx_connection *tc;
523 origLevel = aclient->initializationState;
526 tc = aclient->conns[apos];
527 if (tc && rx_ConnError(tc)) {
528 aclient->conns[apos] = (tc = ubik_RefreshConn(tc));
534 /* now see if we can find the sync site host */
535 code = VOTE_GetSyncSite(tc, &newHost);
536 if (aclient->initializationState != origLevel) {
537 return -1; /* somebody did a ubik_ClientInit */
540 if (!code && newHost) {
541 newHost = htonl(newHost); /* convert back to network order */
544 * position count at the appropriate slot in the client
545 * structure and retry. If we can't find in slot, we'll just
546 * continue through the whole list
548 for (i = 0; i < MAXSERVERS; i++) {
549 rxp = rx_PeerOf(aclient->conns[i]);
550 thisHost = rx_HostOf(rxp);
553 } else if (thisHost == newHost) {
554 return i; /* we were told to use this one */
565 * \brief Create an internal version of ubik_CallIter that takes an additional
566 * parameter - to indicate whether the ubik client handle has already
570 CallIter(int (*aproc) (), register struct ubik_client *aclient,
571 afs_int32 aflags, int *apos, long p1, long p2, long p3, long p4,
572 long p5, long p6, long p7, long p8, long p9, long p10, long p11,
573 long p12, long p13, long p14, long p15, long p16, int needlock)
575 register afs_int32 code;
576 struct rx_connection *tc;
580 LOCK_UBIK_CLIENT(aclient);
582 origLevel = aclient->initializationState;
586 while (*apos < MAXSERVERS) {
587 /* tc is the next conn to try */
588 tc = aclient->conns[*apos];
591 UNLOCK_UBIK_CLIENT(aclient);
596 if (rx_ConnError(tc)) {
597 tc = ubik_RefreshConn(tc);
598 aclient->conns[*apos] = tc;
601 if ((aflags & UPUBIKONLY) && (aclient->states[*apos] & CFLastFailed)) {
602 (*apos)++; /* try another one if this server is down */
604 break; /* this is the desired path */
607 if (*apos >= MAXSERVERS) {
609 UNLOCK_UBIK_CLIENT(aclient);
615 (*aproc) (tc, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13,
617 if (aclient->initializationState != origLevel) {
619 UNLOCK_UBIK_CLIENT(aclient);
621 return code; /* somebody did a ubik_ClientInit */
624 /* what should I do in case of UNOQUORUM ? */
626 aclient->states[*apos] |= CFLastFailed; /* network errors */
628 /* either misc ubik code, or misc application code or success. */
629 aclient->states[*apos] &= ~CFLastFailed; /* operation worked */
634 UNLOCK_UBIK_CLIENT(aclient);
640 * \brief Call this instead of stub and we'll guarantee to find a host that's up.
642 * \todo In the future, we should also put in a protocol to find the sync site.
645 ubik_Call_New(int (*aproc) (), register struct ubik_client *aclient,
646 afs_int32 aflags, long p1, long p2, long p3, long p4, long p5,
647 long p6, long p7, long p8, long p9, long p10, long p11,
648 long p12, long p13, long p14, long p15, long p16)
650 afs_int32 code, rcode;
657 LOCK_UBIK_CLIENT(aclient);
660 origLevel = aclient->initializationState;
662 /* Do two passes. First pass only checks servers known running */
663 for (aflags |= UPUBIKONLY, pass = 0; pass < 2;
664 pass++, aflags &= ~UPUBIKONLY) {
669 CallIter(aproc, aclient, aflags, &count, p1, p2, p3, p4, p5,
670 p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16,
672 if (code && (aclient->initializationState != origLevel)) {
675 if (code == UNOSERVERS) {
678 rcode = code; /* remember code from last good call */
680 if (code == UNOTSYNC) { /* means this requires a sync site */
681 if (aclient->conns[3]) { /* don't bother unless 4 or more srv */
682 temp = try_GetSyncSite(aclient, count);
683 if (aclient->initializationState != origLevel) {
684 goto restart; /* somebody did a ubik_ClientInit */
686 if ((temp >= 0) && ((temp > count) || (stepBack++ <= 2))) {
687 count = temp; /* generally try to make progress */
690 } else if ((code >= 0) && (code != UNOQUORUM)) {
691 UNLOCK_UBIK_CLIENT(aclient);
692 return code; /* success or global error condition */
696 UNLOCK_UBIK_CLIENT(aclient);
701 * \brief This is part of an iterator. It doesn't handle finding sync sites.
704 ubik_CallIter(int (*aproc) (), struct ubik_client *aclient,
705 afs_int32 aflags, int *apos, long p1, long p2,
706 long p3, long p4, long p5, long p6, long p7,
707 long p8, long p9, long p10, long p11, long p12,
708 long p13, long p14, long p15, long p16)
710 return CallIter(aproc, aclient, aflags, apos, p1, p2, p3, p4, p5, p6, p7,
711 p8, p9, p10, p11, p12, p13, p14, p15, p16, NEED_LOCK);