libafs: add replicated connection pool

keep pool of connections to use for replicated volumes,
so we can have a separate idle time setting

Change-Id: I61ed62c652c924b33fde920fac766c4ca0043826
Reviewed-on: http://gerrit.openafs.org/6546
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Derrick Brashear <shadow@dementix.org>

diff --git a/src/afs/afs.h b/src/afs/afs.h
index 4c4873b..9a396a4 100644 (file)
--- a/src/afs/afs.h
+++ b/src/afs/afs.h
@@ -96,12 +96,14 @@ extern int afs_shuttingdown;
 #define        AFS_RXDEADTIME  50
 #define AFS_HARDDEADTIME       120
 #define        AFS_IDLEDEADTIME        1200
+#define AFS_IDLEDEADTIME_REP    180 /* more than fs's cb dead time */
 #define AFS_BLKBITS    12
 #define AFS_BLKSIZE    (1 << AFS_BLKBITS)
 
 extern afs_int32 afs_rx_deadtime;
 extern afs_int32 afs_rx_harddead;
 extern afs_int32 afs_rx_idledead;
+extern afs_int32 afs_rx_idledead_rep;
 
 struct sysname_info {
     char *name;
@@ -380,6 +382,8 @@ struct unixuser {
 struct sa_conn_vector;
 typedef struct sa_conn_vector * p_sa_conn_vector; /* forward decl */
 
+#define CONN_REPLICATED 0x1
+
 struct afs_conn {
     int refCount;
     int activated;
@@ -403,6 +407,7 @@ struct sa_conn_vector {
     struct srvAddr *srvr;      /* server associated with this conn */
     short refCount;            /* reference count for allocation */
     unsigned short port;       /* port associated with this connection */
+    int flags;
 
     /* next connection to return when all in cvec are fully utilized */
     int select_index; 

diff --git a/src/afs/afs_analyze.c b/src/afs/afs_analyze.c
index 6950c96..2f6e400 100644 (file)
--- a/src/afs/afs_analyze.c
+++ b/src/afs/afs_analyze.c
@@ -115,7 +115,7 @@ VLDB_Same(struct VenusFid *afid, struct vrequest *areq)
        VSleep(2);              /* Better safe than sorry. */
        tconn =
            afs_ConnByMHosts(tcell->cellHosts, tcell->vlport, tcell->cellNum,
-                            &treq, SHARED_LOCK, &rxconn);
+                            &treq, SHARED_LOCK, 0, &rxconn);
        if (tconn) {
            if ( tconn->parent->srvr->server->flags & SNO_LHOSTS) {
                type = 0;

diff --git a/src/afs/afs_call.c b/src/afs/afs_call.c
index 34b0655..0e7feb2 100644 (file)
--- a/src/afs/afs_call.c
+++ b/src/afs/afs_call.c
@@ -74,6 +74,7 @@ char afs_cachebasedir[1024];
 afs_int32 afs_rx_deadtime = AFS_RXDEADTIME;
 afs_int32 afs_rx_harddead = AFS_HARDDEADTIME;
 afs_int32 afs_rx_idledead = AFS_IDLEDEADTIME;
+afs_int32 afs_rx_idledead_rep = AFS_IDLEDEADTIME_REP;
 
 static int afscall_set_rxpck_received = 0;
 

diff --git a/src/afs/afs_conn.c b/src/afs/afs_conn.c
index 957365b..0bfb526 100644 (file)
--- a/src/afs/afs_conn.c
+++ b/src/afs/afs_conn.c
@@ -259,6 +259,7 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq,
     int notbusy;
     int i;
     struct srvAddr *sa1p;
+    afs_int32 replicated = -1; /* a single RO will increment to 0 */
 
     *rxconn = NULL;
 
@@ -295,6 +296,8 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq,
      */
     for (notbusy = not_busy; (!lowp && (notbusy <= end_not_busy)); notbusy++) {
        for (i = 0; i < AFS_MAXHOSTS && tv->serverHost[i]; i++) {
+           if (tv->states & VRO)
+               replicated++;
            if (((areq->tokenError > 0)||(areq->idleError > 0))
                && (areq->skipserver[i] == 1))
                continue;
@@ -316,12 +319,20 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq,
            }
        }
     }
+    if ((replicated == -1) && (tv->states & VRO)) {
+       for (i = 0; i < AFS_MAXHOSTS && tv->serverHost[i]; i++) {
+           if (tv->states & VRO)
+               replicated++;
+       }
+    } else
+       replicated = 0;
+
     afs_PutVolume(tv, READ_LOCK);
 
     if (lowp) {
        tu = afs_GetUser(areq->uid, afid->Cell, SHARED_LOCK);
        tconn = afs_ConnBySA(lowp, fsport, afid->Cell, tu, 0 /*!force */ ,
-                            1 /*create */ , locktype, rxconn);
+                            1 /*create */ , locktype, replicated, rxconn);
 
        afs_PutUser(tu, SHARED_LOCK);
     }
@@ -339,6 +350,7 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq,
  * @param tu Connect as this user.
  * @param force_if_down
  * @param create
+ * @param replicated
  * @param locktype Specifies type of lock to be used for this function.
  *
  * @return The new connection.
@@ -346,7 +358,8 @@ afs_Conn(struct VenusFid *afid, struct vrequest *areq,
 struct afs_conn *
 afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell,
             struct unixuser *tu, int force_if_down, afs_int32 create,
-            afs_int32 locktype, struct rx_connection **rxconn)
+            afs_int32 locktype, afs_int32 replicated,
+            struct rx_connection **rxconn)
 {
     int glocked, foundvec;
     struct afs_conn *tc = NULL;
@@ -354,6 +367,7 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell,
     struct rx_securityClass *csec; /*Security class object */
     int isec; /*Security index */
     int service;
+    int isrep = (replicated > 0)?CONN_REPLICATED:0;
 
     *rxconn = NULL;
 
@@ -361,7 +375,8 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell,
     ObtainSharedLock(&afs_xconn, 15);
     foundvec = 0;
     for (tcv = sap->conns; tcv; tcv = tcv->next) {
-        if (tcv->user == tu && tcv->port == aport) {
+        if (tcv->user == tu && tcv->port == aport &&
+           (isrep == (tcv->flags & CONN_REPLICATED))) {
             /* return most eligible conn */
             if (!foundvec)
                 foundvec = 1;
@@ -398,6 +413,8 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell,
         tcv->port = aport;
         tcv->srvr = sap;
         tcv->next = sap->conns;
+       if (isrep)
+           tcv->flags |= CONN_REPLICATED;
         sap->conns = tcv;
 
         /* all struct afs_conn ptrs come from here */
@@ -462,14 +479,17 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell,
        }
 
         /* Setting idle dead time to non-zero activates RX_CALL_IDLE errors. */
-       rx_SetConnIdleDeadTime(tc->id, afs_rx_idledead);
+       if (isrep)
+           rx_SetConnIdleDeadTime(tc->id, afs_rx_idledead_rep);
+       else
+           rx_SetConnIdleDeadTime(tc->id, afs_rx_idledead);
 
        /*
         * Only do this for the base connection, not per-user.
         * Will need to be revisited if/when CB gets security.
         */
        if ((isec == 0) && (service != 52) && !(tu->states & UTokensBad) &&
-           (tu->viceId == UNDEFVID)
+           (tu->viceId == UNDEFVID) && (isrep == 0)
 #ifndef UKERNEL /* ukernel runs as just one uid anyway */
            && (tu->uid == 0)
 #endif
@@ -502,13 +522,14 @@ afs_ConnBySA(struct srvAddr *sap, unsigned short aport, afs_int32 acell,
  * @param areq The request.
  * @param aforce Force connection?
  * @param locktype Type of lock to be used.
+ * @param replicated
  *
  * @return The established connection.
  */
 struct afs_conn *
 afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell,
               struct vrequest *areq, int aforce, afs_int32 locktype,
-              struct rx_connection **rxconn)
+              afs_int32 replicated, struct rx_connection **rxconn)
 {
     struct unixuser *tu;
     struct afs_conn *tc = NULL;
@@ -534,7 +555,7 @@ afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell,
     for (sa = aserver->addr; sa; sa = sa->next_sa) {
        tc = afs_ConnBySA(sa, aport, acell, tu, aforce,
                          0 /*don't create one */ ,
-                         locktype, rxconn);
+                         locktype, replicated, rxconn);
        if (tc)
            break;
     }
@@ -543,7 +564,7 @@ afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell,
        for (sa = aserver->addr; sa; sa = sa->next_sa) {
            tc = afs_ConnBySA(sa, aport, acell, tu, aforce,
                              1 /*create one */ ,
-                             locktype, rxconn);
+                             locktype, replicated, rxconn);
            if (tc)
                break;
        }
@@ -564,13 +585,15 @@ afs_ConnByHost(struct server *aserver, unsigned short aport, afs_int32 acell,
  * @param acell The cell where all of this happens.
  * @param areq The request.
  * @param locktype Type of lock to be used.
+ * @param replicated
  *
  * @return The established connection or NULL.
  */
 struct afs_conn *
 afs_ConnByMHosts(struct server *ahosts[], unsigned short aport,
                 afs_int32 acell, struct vrequest *areq,
-                afs_int32 locktype, struct rx_connection **rxconn)
+                afs_int32 locktype, afs_int32 replicated,
+                struct rx_connection **rxconn)
 {
     afs_int32 i;
     struct afs_conn *tconn;
@@ -583,7 +606,8 @@ afs_ConnByMHosts(struct server *ahosts[], unsigned short aport,
     for (i = 0; i < AFS_MAXCELLHOSTS; i++) {
        if ((ts = ahosts[i]) == NULL)
            break;
-       tconn = afs_ConnByHost(ts, aport, acell, areq, 0, locktype, rxconn);
+       tconn = afs_ConnByHost(ts, aport, acell, areq, 0, locktype,
+                              replicated, rxconn);
        if (tconn) {
            return tconn;
        }

diff --git a/src/afs/afs_pioctl.c b/src/afs/afs_pioctl.c
index d1af181..5cbbdb0 100644 (file)
--- a/src/afs/afs_pioctl.c
+++ b/src/afs/afs_pioctl.c
@@ -5181,7 +5181,7 @@ DECL_PIOCTL(PCallBackAddr)
        /* get a connection, even if host is down; bumps conn ref count */
        tu = afs_GetUser(areq->uid, ts->cell->cellNum, SHARED_LOCK);
        tc = afs_ConnBySA(sa, ts->cell->fsport, ts->cell->cellNum, tu,
-                         1 /*force */ , 1 /*create */ , SHARED_LOCK, &rxconn);
+                         1 /*force */ , 1 /*create */ , SHARED_LOCK, 0, &rxconn);
        afs_PutUser(tu, SHARED_LOCK);
        if (!tc)
            continue;

diff --git a/src/afs/afs_prototypes.h b/src/afs/afs_prototypes.h
index 940df15..71ef0a6 100644 (file)
--- a/src/afs/afs_prototypes.h
+++ b/src/afs/afs_prototypes.h
@@ -174,17 +174,19 @@ extern struct afs_conn *afs_Conn(struct VenusFid *afid,
 extern struct afs_conn *afs_ConnBySA(struct srvAddr *sap, unsigned short aport,
                                 afs_int32 acell, struct unixuser *tu,
                                 int force_if_down, afs_int32 create,
-                                afs_int32 locktype,
+                                afs_int32 locktype, afs_int32 replicated,
                                 struct rx_connection **rxconn);
 extern struct afs_conn *afs_ConnByMHosts(struct server *ahosts[],
                                     unsigned short aport, afs_int32 acell,
                                     struct vrequest *areq,
                                     afs_int32 locktype,
+                                    afs_int32 replicated,
                                     struct rx_connection **rxconn);
 extern struct afs_conn *afs_ConnByHost(struct server *aserver,
                                   unsigned short aport, afs_int32 acell,
                                   struct vrequest *areq, int aforce,
                                   afs_int32 locktype,
+                                  afs_int32 replicated,
                                   struct rx_connection **rxconn);
 extern void afs_PutConn(struct afs_conn *ac, struct rx_connection *rxconn,
                         afs_int32 locktype);

diff --git a/src/afs/afs_server.c b/src/afs/afs_server.c
index d8c4bbf..44f0f4e 100644 (file)
--- a/src/afs/afs_server.c
+++ b/src/afs/afs_server.c
@@ -291,7 +291,8 @@ CheckVLServer(struct srvAddr *sa, struct vrequest *areq)
        return;                 /* can't do much */
 
     tc = afs_ConnByHost(aserver, aserver->cell->vlport,
-                       aserver->cell->cellNum, areq, 1, SHARED_LOCK, &rxconn);
+                       aserver->cell->cellNum, areq, 1, SHARED_LOCK, 0,
+                       &rxconn);
     if (!tc)
        return;
     rx_SetConnDeadTime(rxconn, 3);
@@ -710,7 +711,8 @@ afs_LoopServers(int adown, struct cell *acellp, int vlalso,
        /* get a connection, even if host is down; bumps conn ref count */
        tu = afs_GetUser(treq.uid, ts->cell->cellNum, SHARED_LOCK);
        tc = afs_ConnBySA(sa, ts->cell->fsport, ts->cell->cellNum, tu,
-                         1 /*force */ , 1 /*create */ , SHARED_LOCK, &rxconn);
+                         1 /*force */ , 1 /*create */ , SHARED_LOCK, 0,
+                         &rxconn);
        afs_PutUser(tu, SHARED_LOCK);
        if (!tc)
            continue;
@@ -1549,8 +1551,7 @@ afs_GetCapabilities(struct server *ts)
     if ( !tu )
        return;
     tc = afs_ConnBySA(ts->addr, ts->cell->fsport, ts->cell->cellNum, tu, 0, 1,
-                                                               SHARED_LOCK,
-                                                                &rxconn);
+                     SHARED_LOCK, 0, &rxconn);
     afs_PutUser(tu, SHARED_LOCK);
     if ( !tc )
        return;

diff --git a/src/afs/afs_vcache.c b/src/afs/afs_vcache.c
index 08d54f5..7428c94 100644 (file)
--- a/src/afs/afs_vcache.c
+++ b/src/afs/afs_vcache.c
@@ -456,7 +456,7 @@ afs_FlushVCBs(afs_int32 lockit)
                    for (safety3 = 0; safety3 < AFS_MAXHOSTS * 2; safety3++) {
                        tc = afs_ConnByHost(tsp, tsp->cell->fsport,
                                            tsp->cell->cellNum, &treq, 0,
-                                           SHARED_LOCK, &rxconn);
+                                           SHARED_LOCK, 0, &rxconn);
                        if (tc) {
                            XSTATS_START_TIME
                                (AFS_STATS_FS_RPCIDX_GIVEUPCALLBACKS);

diff --git a/src/afs/afs_volume.c b/src/afs/afs_volume.c
index e12d3e3..2602c3e 100644 (file)
--- a/src/afs/afs_volume.c
+++ b/src/afs/afs_volume.c
@@ -766,7 +766,7 @@ afs_NewVolumeByName(char *aname, afs_int32 acell, int agood,
     do {
        tconn =
            afs_ConnByMHosts(tcell->cellHosts, tcell->vlport, tcell->cellNum,
-                            &treq, SHARED_LOCK, &rxconn);
+                            &treq, SHARED_LOCK, 0, &rxconn);
        if (tconn) {
            if (tconn->parent->srvr->server->flags & SNO_LHOSTS) {
                type = 0;
@@ -1114,7 +1114,7 @@ LockAndInstallUVolumeEntry(struct volume *av, struct uvldbentry *ve, int acell,
                    tconn =
                        afs_ConnByMHosts(tcell->cellHosts, tcell->vlport,
                                         tcell->cellNum, areq, SHARED_LOCK,
-                                        &rxconn);
+                                        0, &rxconn);
                    if (tconn) {
                        RX_AFS_GUNLOCK();
                        code =