diff options
| -rw-r--r-- | libglusterfs/src/latency.c | 12 | ||||
| -rw-r--r-- | libglusterfs/src/mem-types.h | 1 | ||||
| -rw-r--r-- | libglusterfs/src/timespec.c | 12 | ||||
| -rw-r--r-- | libglusterfs/src/timespec.h | 3 | ||||
| -rw-r--r-- | libglusterfs/src/xlator.h | 1 | ||||
| -rw-r--r-- | rpc/rpc-lib/src/rpc-clnt-ping.c | 59 | ||||
| -rw-r--r-- | rpc/rpc-lib/src/rpc-clnt.h | 1 | ||||
| -rw-r--r-- | rpc/xdr/src/glusterfs-fops.x | 1 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr-common.c | 368 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr-mem-types.h | 3 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr-self-heald.h | 1 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr.c | 86 | ||||
| -rw-r--r-- | xlators/cluster/afr/src/afr.h | 12 | ||||
| -rw-r--r-- | xlators/features/changelog/lib/src/gf-changelog-rpc.c | 1 | ||||
| -rw-r--r-- | xlators/features/changelog/src/changelog-ev-handle.c | 2 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-volgen.c | 22 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-volume-set.c | 25 | ||||
| -rwxr-xr-x | xlators/mount/fuse/utils/mount.glusterfs.in | 24 | ||||
| -rw-r--r-- | xlators/protocol/client/src/client-handshake.c | 2 | ||||
| -rw-r--r-- | xlators/protocol/client/src/client.c | 98 | 
20 files changed, 622 insertions, 112 deletions
diff --git a/libglusterfs/src/latency.c b/libglusterfs/src/latency.c index 611615949fa..3399cc7c297 100644 --- a/libglusterfs/src/latency.c +++ b/libglusterfs/src/latency.c @@ -21,6 +21,7 @@  #include "statedump.h"  #include "libglusterfs-messages.h" +static int gf_set_fop_from_fn_pointer_warning;  void  gf_set_fop_from_fn_pointer (call_frame_t *frame, struct xlator_fops *fops, void *fn)  { @@ -108,8 +109,15 @@ gf_set_fop_from_fn_pointer (call_frame_t *frame, struct xlator_fops *fops, void                  fop = GF_FOP_READDIRP;          else if (fops->getspec == *(fop_getspec_t *)&fn)                  fop = GF_FOP_GETSPEC; -        else -                fop = -1; +        else if (fops->ipc == *(fop_ipc_t *)&fn) +                fop = GF_FOP_IPC; +        else { +                fop = GF_FOP_NULL; +                GF_LOG_OCCASIONALLY(gf_set_fop_from_fn_pointer_warning, +                                    "latency", +                                    GF_LOG_WARNING, +                                    "Unknown FOP type"); +        }          frame->op   = fop;  } diff --git a/libglusterfs/src/mem-types.h b/libglusterfs/src/mem-types.h index afa52d8bc45..fc7bf9e5996 100644 --- a/libglusterfs/src/mem-types.h +++ b/libglusterfs/src/mem-types.h @@ -168,6 +168,7 @@ enum gf_common_mem_types_ {          /*lock migration*/          gf_common_mt_lock_mig,          gf_common_mt_pthread_t, +        gf_common_ping_local_t,          gf_common_mt_end  };  #endif diff --git a/libglusterfs/src/timespec.c b/libglusterfs/src/timespec.c index f7b2bea2f30..903303d1380 100644 --- a/libglusterfs/src/timespec.c +++ b/libglusterfs/src/timespec.c @@ -60,3 +60,15 @@ void timespec_adjust_delta (struct timespec *ts, struct timespec delta)          ts->tv_sec += ((ts->tv_nsec + delta.tv_nsec) / 1000000000);          ts->tv_sec += delta.tv_sec;  } + +void timespec_sub (const struct timespec *begin, const struct timespec *end, +                   struct timespec *res) +{ +        if (end->tv_nsec < begin->tv_nsec) { +                res->tv_sec = end->tv_sec - begin->tv_sec - 1; +                res->tv_nsec = end->tv_nsec + 1000000000 - begin->tv_nsec; +        } else { +                res->tv_sec = end->tv_sec - begin->tv_sec; +                res->tv_nsec = end->tv_nsec - begin->tv_nsec; +        } +} diff --git a/libglusterfs/src/timespec.h b/libglusterfs/src/timespec.h index f37194b97cf..9c393ee7166 100644 --- a/libglusterfs/src/timespec.h +++ b/libglusterfs/src/timespec.h @@ -20,5 +20,8 @@  void timespec_now (struct timespec *ts);  void timespec_adjust_delta (struct timespec *ts, struct timespec delta); +void timespec_sub (const struct timespec *begin, +                   const struct timespec *end, +                   struct timespec *res);  #endif /*  __INCLUDE_TIMESPEC_H__ */ diff --git a/libglusterfs/src/xlator.h b/libglusterfs/src/xlator.h index 70e6f0a108d..571dce2aded 100644 --- a/libglusterfs/src/xlator.h +++ b/libglusterfs/src/xlator.h @@ -927,6 +927,7 @@ struct _xlator {          gf_loglevel_t    loglevel;   /* Log level for translator */ +        int64_t             client_latency;          /* for latency measurement */          fop_latency_t latencies[GF_FOP_MAXVALUE]; diff --git a/rpc/rpc-lib/src/rpc-clnt-ping.c b/rpc/rpc-lib/src/rpc-clnt-ping.c index a7ff866ac99..91a2ab84140 100644 --- a/rpc/rpc-lib/src/rpc-clnt-ping.c +++ b/rpc/rpc-lib/src/rpc-clnt-ping.c @@ -18,6 +18,7 @@  #include "mem-pool.h"  #include "xdr-rpc.h"  #include "rpc-common-xdr.h" +#include "timespec.h"  char *clnt_ping_procs[GF_DUMP_MAXVALUE] = { @@ -30,6 +31,11 @@ struct rpc_clnt_program clnt_ping_prog = {          .procnames = clnt_ping_procs,  }; +struct ping_local { +    struct rpc_clnt *rpc; +    struct timespec submit_time; +}; +  /* Must be called under conn->lock */  static int  __rpc_clnt_rearm_ping_timer (struct rpc_clnt *rpc, gf_timer_cbk_t cbk) @@ -170,11 +176,16 @@ int  rpc_clnt_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,                     void *myframe)  { -        struct rpc_clnt       *rpc     = NULL; +        struct ping_local     *local   = NULL;          xlator_t              *this    = NULL;          rpc_clnt_connection_t *conn    = NULL; +          call_frame_t          *frame   = NULL;          struct timespec       timeout  = {0, }; +        struct timespec       now; +        struct timespec       delta; +        int64_t               latency_msec = 0; +        int                   ret = 0;          int                   unref    = 0;          if (!myframe) { @@ -185,14 +196,28 @@ rpc_clnt_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,          frame = myframe;          this = frame->this; -        rpc  = frame->local; -        frame->local = NULL; /* Prevent STACK_DESTROY from segfaulting */ -        conn = &rpc->conn; +        local = frame->local; +        conn = &local->rpc->conn;          pthread_mutex_lock (&conn->lock);          { +                timespec_now (&now); +                timespec_sub (&local->submit_time, &now, &delta); +                latency_msec = delta.tv_sec * 1000 + delta.tv_nsec / 1000000; + +                this->client_latency = latency_msec; +                gf_log (THIS->name, GF_LOG_DEBUG, +                        "Ping latency is %" PRIu64 "ms", +                        latency_msec); + +                ret = local->rpc->notifyfn (local->rpc, NULL, RPC_CLNT_PING, +                                            NULL); +                if (ret) +                        gf_log (this->name, GF_LOG_WARNING, +                                "RPC_CLNT_PING notify failed"); +                  if (req->rpc_status == -1) { -                        unref = rpc_clnt_remove_ping_timer_locked (rpc); +                        unref = rpc_clnt_remove_ping_timer_locked (local->rpc);                          if (unref) {                                  gf_log (this->name, GF_LOG_WARNING,                                          "socket or ib related error"); @@ -207,8 +232,8 @@ rpc_clnt_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,                          goto unlock;                  } -                unref = rpc_clnt_remove_ping_timer_locked (rpc); -                if (__rpc_clnt_rearm_ping_timer (rpc, +                unref = rpc_clnt_remove_ping_timer_locked (local->rpc); +                if (__rpc_clnt_rearm_ping_timer (local->rpc,                                                   rpc_clnt_start_ping) == -1) {                          gf_log (this->name, GF_LOG_WARNING,                                  "failed to set the ping timer"); @@ -219,10 +244,13 @@ unlock:          pthread_mutex_unlock (&conn->lock);  out:          if (unref) -                rpc_clnt_unref (rpc); +                rpc_clnt_unref (local->rpc); -        if (frame) +        if (frame) { +                GF_FREE (frame->local); +                frame->local = NULL;                  STACK_DESTROY (frame->root); +        }          return 0;  } @@ -232,18 +260,27 @@ rpc_clnt_ping (struct rpc_clnt *rpc)          call_frame_t *frame = NULL;          int32_t       ret   = -1;          rpc_clnt_connection_t *conn = NULL; +        struct ping_local *local = NULL;          conn = &rpc->conn; +        local = GF_MALLOC (sizeof(struct ping_local), gf_common_ping_local_t); +        if (!local) +                return ret;          frame = create_frame (THIS, THIS->ctx->pool); -        if (!frame) +        if (!frame) { +                GF_FREE (local);                  return ret; +        } -        frame->local = rpc; +        local->rpc = rpc; +        timespec_now (&local->submit_time); +        frame->local = local;          ret = rpc_clnt_submit (rpc, &clnt_ping_prog,                                 GF_DUMP_PING, rpc_clnt_ping_cbk, NULL, 0,                                 NULL, 0, NULL, frame, NULL, 0, NULL, 0, NULL);          if (ret) { +                /* FIXME: should we free the frame here? Methinks so! */                  gf_log (THIS->name, GF_LOG_ERROR,                          "failed to start ping timer");          } diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h index f84b4cbf806..df84d5966d8 100644 --- a/rpc/rpc-lib/src/rpc-clnt.h +++ b/rpc/rpc-lib/src/rpc-clnt.h @@ -19,6 +19,7 @@  typedef enum {          RPC_CLNT_CONNECT,          RPC_CLNT_DISCONNECT, +        RPC_CLNT_PING,          RPC_CLNT_MSG,          RPC_CLNT_DESTROY  } rpc_clnt_event_t; diff --git a/rpc/xdr/src/glusterfs-fops.x b/rpc/xdr/src/glusterfs-fops.x index 8462dcc258a..5ec8109d828 100644 --- a/rpc/xdr/src/glusterfs-fops.x +++ b/rpc/xdr/src/glusterfs-fops.x @@ -84,6 +84,7 @@ enum glusterfs_event_t {          GF_EVENT_UPCALL,          GF_EVENT_SCRUB_STATUS,          GF_EVENT_SOME_CHILD_DOWN, +        GF_EVENT_CHILD_PING,          GF_EVENT_MAXVAL  }; diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c index 795e43b8bd4..c2e95953a7f 100644 --- a/xlators/cluster/afr/src/afr-common.c +++ b/xlators/cluster/afr/src/afr-common.c @@ -2259,7 +2259,6 @@ unwind:          return 0;  } -  int  afr_lookup_entry_heal (call_frame_t *frame, xlator_t *this)  { @@ -2953,7 +2952,7 @@ afr_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,          }          UNLOCK (&frame->lock); -	call_count = afr_frame_return (frame); +        call_count = afr_frame_return (frame);  	if (call_count == 0)  		AFR_STACK_UNWIND (flush, frame, local->op_ret, @@ -4201,25 +4200,319 @@ __get_heard_from_all_status (xlator_t *this)          return heard_from_all;  } +int +find_best_down_child (xlator_t *this) +{ +        afr_private_t   *priv               = NULL; +        int             i                   = -1; +        int32_t         best_child          = -1; +        int64_t         best_latency        = INT64_MAX; + +        priv = this->private; + +        for (i = 0; i < priv->child_count; i++) { +                if (priv->child_up[i] && +                    priv->child_latency[i] >= 0.0 && +                    priv->child_latency[i] < best_latency) { +                        best_child = i; +                        best_latency = priv->child_latency[i]; +                } +        } +        if (best_child >= 0) { +                gf_log (this->name, GF_LOG_DEBUG, "Found best down child (%d) " +                        "@ %ld ms latency", best_child, best_latency); +        } +        return best_child; +} + +int +find_worst_up_child (xlator_t *this) +{ +        afr_private_t   *priv               = NULL; +        int             i                   = -1; +        int32_t         worst_child         = -1; +        int64_t         worst_latency       = INT64_MIN; + +        priv = this->private; + +        for (i = 0; i < priv->child_count; i++) { +                if (priv->child_up[i] && +                    priv->child_latency[i] >= 0.0 && +                    priv->child_latency[i] > worst_latency) { +                        worst_child = i; +                        worst_latency = priv->child_latency[i]; +                } +        } +        if (worst_child >= 0) { +                gf_log (this->name, GF_LOG_DEBUG, "Found worst up child (%d)" +                        " @ %ld ms latency", worst_child, worst_latency); +        } +        return worst_child; +} + +void +_afr_handle_ping_event (xlator_t *this, xlator_t *child_xlator, +                const int idx, int64_t halo_max_latency_msec, int32_t *event, +                int64_t *child_latency_msec) +{ +        afr_private_t   *priv               = NULL; +        int             i                   = -1; +        int             up_children         = 0; + +        priv = this->private; + +        *child_latency_msec = child_xlator->client_latency; +        priv->child_latency[idx] = *child_latency_msec; +        gf_log (child_xlator->name, GF_LOG_DEBUG, "Client ping @ %ld ms", +                *child_latency_msec); + +        for (i = 0; i < priv->child_count; i++) +                if (priv->child_up[i] == 1) +                        up_children++; + +        if (*child_latency_msec > halo_max_latency_msec && +            priv->child_up[idx] == 1 && +            up_children > priv->halo_min_replicas) { +                if ((up_children - 1) < +                    priv->halo_min_replicas) { +                        gf_log (child_xlator->name, GF_LOG_INFO, +                               "Overriding halo threshold, " +                               "min replicas: %d", +                               priv->halo_min_replicas); +                } else { +                        gf_log (child_xlator->name, GF_LOG_INFO, +                                "Child latency (%ld ms) " +                                "exceeds halo threshold (%ld), " +                                "marking child down.", +                                *child_latency_msec, +                                halo_max_latency_msec); +                        *event = GF_EVENT_CHILD_DOWN; +                } +        } else if (*child_latency_msec < halo_max_latency_msec && +                   priv->child_up[idx] == 0) { +                if (up_children < priv->halo_max_replicas) { +                        gf_log (child_xlator->name, GF_LOG_INFO, +                                "Child latency (%ld ms) " +                                "below halo threshold (%ld), " +                                "marking child up.", +                                *child_latency_msec, +                                halo_max_latency_msec); +                        *event = GF_EVENT_CHILD_UP; +                } else { +                        gf_log (child_xlator->name, GF_LOG_INFO, +                            "Not marking child %d up, " +                            "max replicas (%d) reached.", idx, +                            priv->halo_max_replicas); +                } +        } +} + +void +_afr_handle_child_up_event (xlator_t *this, xlator_t *child_xlator, +                const int idx, int64_t halo_max_latency_msec, +                int32_t *event, int32_t *call_psh, int32_t *up_child) +{ +        afr_private_t   *priv               = NULL; +        int             i                   = -1; +        int             up_children         = 0; +        int             worst_up_child      = -1; + +        priv = this->private; + +        /* +         * This only really counts if the child was never up +         * (value = -1) or had been down (value = 0).  See +         * comment at GF_EVENT_CHILD_DOWN for a more detailed +         * explanation. +         */ +        if (priv->child_up[idx] != 1) { +                priv->event_generation++; +        } +        priv->child_up[idx] = 1; + +        *call_psh = 1; +        *up_child = idx; +        for (i = 0; i < priv->child_count; i++) +                if (priv->child_up[i] == 1) +                        up_children++; + +        /* +         * Handle the edge case where we exceed +         * halo_min_replicas and we've got a child which is +         * marked up as it was helping to satisfy the +         * halo_min_replicas even though it's latency exceeds +         * halo_max_latency_msec. +         */ +        if (up_children > priv->halo_min_replicas) { +                worst_up_child = find_worst_up_child (this); +                if (worst_up_child >= 0 && +                    priv->child_latency[worst_up_child] > +                    halo_max_latency_msec) { +                        gf_log (this->name, GF_LOG_DEBUG, +                                "Marking child %d down, " +                                "doesn't meet halo threshold " +                                "(%ld), and > " +                                "halo_min_replicas (%d)", +                                worst_up_child, +                                halo_max_latency_msec, +                                priv->halo_min_replicas); +                        priv->child_up[worst_up_child] = 0; +                        up_children--; +                } +        } +        if (up_children > priv->halo_max_replicas && +            !priv->shd.iamshd) { +                worst_up_child = find_worst_up_child (this); +                if (worst_up_child < 0) { +                        worst_up_child = idx; +                } +                priv->child_up[worst_up_child] = 0; +                up_children--; +                gf_log (this->name, GF_LOG_DEBUG, +                        "Marking child %d down, " +                        "up_children (%d) > " +                        "halo_max_replicas (%d)", +                        worst_up_child, +                        up_children, +                        priv->halo_max_replicas); +        } + +        if (up_children == 1) { +                gf_log (this->name, GF_LOG_INFO, +                        "Subvolume '%s' came back up; " +                        "going online.", +                        child_xlator->name); +        } else { +                *event = GF_EVENT_CHILD_MODIFIED; +        } + +        priv->last_event[idx] = *event; +} + +void +_afr_handle_child_down_event (xlator_t *this, xlator_t *child_xlator, +                int idx, int64_t child_latency_msec, int32_t *event, +                int32_t *call_psh, int32_t *up_child) +{ +        afr_private_t   *priv               = NULL; +        int             i                   = -1; +        int             up_children         = 0; +        int             down_children       = 0; +        int             best_down_child     = -1; + +        priv = this->private; + +        /* +         * If a brick is down when we start, we'll get a +         * CHILD_DOWN to indicate its initial state.  There +         * was never a CHILD_UP in this case, so if we +         * increment "down_count" the difference between than +         * and "up_count" will no longer be the number of +         * children that are currently up.  This has serious +         * implications e.g. for quorum enforcement, so we +         * don't increment these values unless the event +         * represents an actual state transition between "up" +         * (value = 1) and anything else. +         */ +        if (priv->child_up[idx] == 1) { +                priv->event_generation++; +        } + +        /* +         * If this is an _actual_ CHILD_DOWN event, we +         * want to set the child_latency to < 0 to indicate +         * the child is really disconnected. +         */ +        if (child_latency_msec < 0) { +                priv->child_latency[idx] = child_latency_msec; +        } +        priv->child_up[idx] = 0; + +        for (i = 0; i < priv->child_count; i++) +                if (priv->child_up[i] == 1) +                        up_children++; + +        /* +         * Handle the edge case where we need to find the +         * next best child (to mark up) as marking this child +         * down would cause us to fall below halo_min_replicas. +         * We will also force the SHD to heal this child _now_ +         * as we want it to be up to date if we are going to +         * begin using it synchronously. +         */ +        if (up_children < priv->halo_min_replicas) { +                best_down_child = find_best_down_child (this); +                if (best_down_child >= 0) { +                        gf_log (this->name, GF_LOG_DEBUG, +                                "Swapping out child %d for " +                                "child %d to satisfy " +                                "halo_min_replicas (%d).", +                                idx, best_down_child, +                                priv->halo_min_replicas); +                        priv->child_up[best_down_child] = 1; +                        *call_psh = 1; +                        *up_child = best_down_child; +                } +        } + +        for (i = 0; i < priv->child_count; i++) +                if (priv->child_up[i] == 0) +                        down_children++; +        if (down_children == priv->child_count) { +                gf_log (this->name, GF_LOG_ERROR, +                        "All subvolumes are down. Going " +                        "offline until atleast one of them " +                        "comes back up."); +        } else { +                *event = GF_EVENT_CHILD_MODIFIED; +        } +        priv->last_event[idx] = *event; +} + +int64_t +_afr_get_halo_latency (xlator_t *this) +{ +        afr_private_t *priv           = NULL; +        int64_t halo_max_latency_msec = 0; + +        priv = this->private; + +        if (priv->shd.iamshd) { +                halo_max_latency_msec = priv->shd.halo_max_latency_msec; +        } else if (priv->nfsd.iamnfsd) { +                halo_max_latency_msec = +                        priv->nfsd.halo_max_latency_msec; +        } else { +                halo_max_latency_msec = priv->halo_max_latency_msec; +        } +        gf_log (this->name, GF_LOG_DEBUG, "Using halo latency %ld", +                halo_max_latency_msec); +        return halo_max_latency_msec; +} + +  int32_t  afr_notify (xlator_t *this, int32_t event,              void *data, void *data2)  { +        xlator_t        *child_xlator       = NULL;          afr_private_t   *priv               = NULL;          int             i                   = -1; -        int             up_children         = 0; -        int             down_children       = 0;          int             propagate           = 0;          int             had_heard_from_all  = 0;          int             have_heard_from_all = 0;          int             idx                 = -1;          int             ret                 = -1;          int             call_psh            = 0; +        int             up_child            = -1;          dict_t          *input              = NULL;          dict_t          *output             = NULL;          gf_boolean_t    had_quorum          = _gf_false;          gf_boolean_t    has_quorum          = _gf_false; +        int64_t         halo_max_latency_msec = 0; +        int64_t         child_latency_msec   = 0; +        child_xlator = (xlator_t *)data;          priv = this->private;          if (!priv) @@ -4243,7 +4536,7 @@ afr_notify (xlator_t *this, int32_t event,           * subsequent revalidate lookup happens on all the dht's subvolumes           * which triggers afr self-heals if any.           */ -        idx = find_child_index (this, data); +        idx = find_child_index (this, child_xlator);          if (idx < 0) {                  gf_msg (this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_CHILD_UP,                          "Received child_up from invalid subvolume"); @@ -4252,6 +4545,21 @@ afr_notify (xlator_t *this, int32_t event,          had_quorum = priv->quorum_count && afr_has_quorum (priv->child_up,                                                             this); + +        halo_max_latency_msec = _afr_get_halo_latency (this); + +        if (event == GF_EVENT_CHILD_PING) { +                /* Calculates the child latency and sets event +                 */ +                LOCK (&priv->lock); +                { +                        _afr_handle_ping_event (this, child_xlator, idx, +                                halo_max_latency_msec, &event, +                                &child_latency_msec); +                } +                UNLOCK (&priv->lock); +        } +          if (event == GF_EVENT_TRANSLATOR_OP) {                  LOCK (&priv->lock);                  { @@ -4278,52 +4586,15 @@ afr_notify (xlator_t *this, int32_t event,                          propagate = 1;                          break;                  case GF_EVENT_CHILD_UP: -                        /* -                         * This only really counts if the child was never up -                         * (value = -1) or had been down (value = 0).  See -                         * comment at GF_EVENT_CHILD_DOWN for a more detailed -                         * explanation. -                         */ -                        if (priv->child_up[idx] != 1) { -                                priv->event_generation++; -                        } -                        priv->child_up[idx] = 1; - -                        call_psh = 1; -                        up_children = __afr_get_up_children_count (priv); -                        if (up_children == 1) { -                                gf_msg (this->name, GF_LOG_INFO, 0, -                                        AFR_MSG_SUBVOL_UP, -                                        "Subvolume '%s' came back up; " -                                     "going online.", ((xlator_t *)data)->name); -                        } else { -                                event = GF_EVENT_CHILD_MODIFIED; -                        } - -                        priv->last_event[idx] = event; - +                        _afr_handle_child_up_event (this, child_xlator, +                                idx, halo_max_latency_msec, &event, &call_psh, +                                &up_child);                          break;                  case GF_EVENT_CHILD_DOWN: -                        if (priv->child_up[idx] == 1) { -                                priv->event_generation++; -                        } -                        priv->child_up[idx] = 0; - -                        for (i = 0; i < priv->child_count; i++) -                                if (priv->child_up[i] == 0) -                                        down_children++; -                        if (down_children == priv->child_count) { -                                gf_msg (this->name, GF_LOG_ERROR, 0, -                                        AFR_MSG_ALL_SUBVOLS_DOWN, -                                       "All subvolumes are down. Going offline " -                                    "until atleast one of them comes back up."); -                        } else { -                                event = GF_EVENT_SOME_CHILD_DOWN; -                        } - -                        priv->last_event[idx] = event; - +                        _afr_handle_child_down_event (this, child_xlator, idx, +                                child_latency_msec, &event, &call_psh, +                                &up_child);                          break;                  case GF_EVENT_CHILD_CONNECTING: @@ -4350,7 +4621,6 @@ afr_notify (xlator_t *this, int32_t event,                             had come up, propagate CHILD_UP, but only this time                          */                          event = GF_EVENT_CHILD_DOWN; -                        up_children = __afr_get_up_children_count (priv);                          for (i = 0; i < priv->child_count; i++) {                                  if (priv->last_event[i] == GF_EVENT_CHILD_UP) {                                          event = GF_EVENT_CHILD_UP; diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h index 7f7962013d7..c7d6261b110 100644 --- a/xlators/cluster/afr/src/afr-mem-types.h +++ b/xlators/cluster/afr/src/afr-mem-types.h @@ -46,7 +46,8 @@ enum gf_afr_mem_types_ {  	gf_afr_mt_spbc_timeout_t,          gf_afr_mt_spb_status_t,          gf_afr_mt_empty_brick_t, -        gf_afr_mt_end +        gf_afr_mt_child_latency_t, +    gf_afr_mt_end  };  #endif diff --git a/xlators/cluster/afr/src/afr-self-heald.h b/xlators/cluster/afr/src/afr-self-heald.h index c6ac5ebfd1b..4ac1d32f58a 100644 --- a/xlators/cluster/afr/src/afr-self-heald.h +++ b/xlators/cluster/afr/src/afr-self-heald.h @@ -58,6 +58,7 @@ typedef struct {          eh_t                    **statistics;          uint32_t                max_threads;          uint32_t                wait_qlength; +        uint32_t                halo_max_latency_msec;  } afr_self_heald_t; diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c index 6f4783c9213..d1a1e9837da 100644 --- a/xlators/cluster/afr/src/afr.c +++ b/xlators/cluster/afr/src/afr.c @@ -176,6 +176,23 @@ reconfigure (xlator_t *this, dict_t *options)          GF_OPTION_RECONF ("data-self-heal-algorithm",                            priv->data_self_heal_algorithm, options, str, out); +        GF_OPTION_RECONF ("halo-shd-max-latency", +                          priv->shd.halo_max_latency_msec, options, uint32, +                          out); + +        GF_OPTION_RECONF ("halo-nfsd-max-latency", +                          priv->nfsd.halo_max_latency_msec, options, uint32, +                          out); + +        GF_OPTION_RECONF ("halo-max-latency", priv->halo_max_latency_msec, +                          options, uint32, out); + +        GF_OPTION_RECONF ("halo-max-replicas", priv->halo_max_replicas, options, +                              uint32, out); + +        GF_OPTION_RECONF ("halo-min-replicas", priv->halo_min_replicas, options, +                              uint32, out); +          GF_OPTION_RECONF ("read-subvolume", read_subvol, options, xlator, out);          GF_OPTION_RECONF ("read-hash-mode", priv->hash_mode, @@ -396,6 +413,21 @@ init (xlator_t *this)          GF_OPTION_INIT ("entry-self-heal", priv->entry_self_heal, bool, out); +        GF_OPTION_INIT ("halo-shd-max-latency", priv->shd.halo_max_latency_msec, +                        uint32, out); + +        GF_OPTION_INIT ("halo-max-latency", priv->halo_max_latency_msec, +                        uint32, out); +        GF_OPTION_INIT ("halo-max-replicas", priv->halo_max_replicas, uint32, +                        out); +        GF_OPTION_INIT ("halo-min-replicas", priv->halo_min_replicas, uint32, +                        out); + +        GF_OPTION_INIT ("halo-nfsd-max-latency", +                        priv->nfsd.halo_max_latency_msec, uint32, out); + +        GF_OPTION_INIT ("iam-nfs-daemon", priv->nfsd.iamnfsd, bool, out); +          GF_OPTION_INIT ("data-change-log", priv->data_change_log, bool, out);          GF_OPTION_INIT ("metadata-change-log", priv->metadata_change_log, bool, @@ -445,7 +477,12 @@ init (xlator_t *this)          priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,                                      gf_afr_mt_char); -        if (!priv->child_up) { + +        priv->child_latency = GF_CALLOC (sizeof (*priv->child_latency), +                                         child_count, +                                         gf_afr_mt_child_latency_t); + +        if (!priv->child_up || !priv->child_latency) {                  ret = -ENOMEM;                  goto out;          } @@ -663,7 +700,45 @@ struct volume_options options[] = {                           "jobs that can perform parallel heals in the "                           "background."          }, -        { .key  = {"heal-wait-queue-length"}, +        { .key   = {"halo-shd-max-latency"}, +          .type  = GF_OPTION_TYPE_INT, +          .min   = 1, +          .max   = 99999, +          .default_value = "500", +           .description = "Maximum latency for shd halo replication in msec." +        }, +        { .key   = {"halo-nfsd-max-latency"}, +          .type  = GF_OPTION_TYPE_INT, +          .min   = 1, +          .max   = 99999, +          .default_value = "10", +           .description = "Maximum latency for nfsd halo replication in msec." +        }, +        { .key   = {"halo-max-latency"}, +          .type  = GF_OPTION_TYPE_INT, +          .min   = 1, +          .max   = 99999, +          .default_value = "10", +           .description = "Maximum latency for halo replication in msec." +        }, +        { .key   = {"halo-max-replicas"}, +          .type  = GF_OPTION_TYPE_INT, +          .min   = 1, +          .max   = 99999, +          .default_value = "99999", +           .description = "The maximum number of halo replicas; replicas" +                          " beyond this value will be written asynchronously" +                          "via the SHD." +        }, +        { .key   = {"halo-min-replicas"}, +          .type  = GF_OPTION_TYPE_INT, +          .min   = 1, +          .max   = 99999, +          .default_value = "1", +           .description = "The minimmum number of halo replicas, before adding " +                          "out of region replicas." +         }, +         { .key  = {"heal-wait-queue-length"},            .type = GF_OPTION_TYPE_INT,            .min  = 0,            .max  = 10000, /*Around 100MB with sizeof(afr_local_t)= 10496 bytes*/ @@ -802,6 +877,13 @@ struct volume_options options[] = {                           "translator is running as part of self-heal-daemon "                           "or not."          }, +        { .key = {"iam-nfs-daemon"}, +          .type = GF_OPTION_TYPE_BOOL, +          .default_value = "off", +          .description = "This option differentiates if the replicate " +                         "translator is running as part of an NFS daemon " +                         "or not." +        },          { .key = {"quorum-type"},            .type = GF_OPTION_TYPE_STR,            .value = { "none", "auto", "fixed"}, diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h index 4bffc30788a..913f6a7369d 100644 --- a/xlators/cluster/afr/src/afr.h +++ b/xlators/cluster/afr/src/afr.h @@ -60,6 +60,11 @@ typedef enum {          AFR_FAV_CHILD_POLICY_MAX,  } afr_favorite_child_policy; +struct afr_nfsd { +        gf_boolean_t     iamnfsd; +        uint32_t         halo_max_latency_msec; +}; +  typedef struct _afr_private {          gf_lock_t lock;               /* to guard access to child_count, etc */          unsigned int child_count;     /* total number of children   */ @@ -71,6 +76,7 @@ typedef struct _afr_private {          inode_t *root_inode;          unsigned char *child_up; +        int64_t *child_latency;          unsigned char *local;          char **pending_key; @@ -141,8 +147,12 @@ typedef struct _afr_private {          gf_boolean_t           ensure_durability;          char                   *sh_domain;  	char                   *afr_dirty; +        uint32_t               halo_max_latency_msec; +        uint32_t               halo_max_replicas; +        uint32_t               halo_min_replicas; -	afr_self_heald_t       shd; +        afr_self_heald_t       shd; +        struct afr_nfsd        nfsd;          gf_boolean_t           consistent_metadata;          uint64_t               spb_choice_timeout; diff --git a/xlators/features/changelog/lib/src/gf-changelog-rpc.c b/xlators/features/changelog/lib/src/gf-changelog-rpc.c index 270632bc71b..2eb3a9f9149 100644 --- a/xlators/features/changelog/lib/src/gf-changelog-rpc.c +++ b/xlators/features/changelog/lib/src/gf-changelog-rpc.c @@ -26,6 +26,7 @@ gf_changelog_rpc_notify (struct rpc_clnt *rpc,          case RPC_CLNT_DISCONNECT:          case RPC_CLNT_MSG:          case RPC_CLNT_DESTROY: +        case RPC_CLNT_PING:                  break;          } diff --git a/xlators/features/changelog/src/changelog-ev-handle.c b/xlators/features/changelog/src/changelog-ev-handle.c index 77637c7beec..459d173db7f 100644 --- a/xlators/features/changelog/src/changelog-ev-handle.c +++ b/xlators/features/changelog/src/changelog-ev-handle.c @@ -180,6 +180,8 @@ changelog_rpc_notify (struct rpc_clnt *rpc,                  /* Free up mydata */                  changelog_rpc_clnt_unref (crpc);                  break; +        case RPC_CLNT_PING: +                break;          }          return 0; diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c index b6b4914d422..7f7ec7232c1 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.c +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c @@ -4731,6 +4731,24 @@ out:          return ret;  } +static int +volgen_graph_set_iam_nfsd (const volgen_graph_t *graph) +{ +        xlator_t        *trav; +        int             ret = 0; + +        for (trav = first_of ((volgen_graph_t *)graph); trav; +                        trav = trav->next) { +                if (strcmp (trav->type, "cluster/replicate") != 0) +                        continue; + +                ret = xlator_set_option (trav, "iam-nfs-daemon", "yes"); +                if (ret) +                        break; +        } +        return ret; +} +  /* builds a graph for nfs server role, with option overrides in mod_dict */  int  build_nfs_graph (volgen_graph_t *graph, dict_t *mod_dict) @@ -4869,6 +4887,10 @@ build_nfs_graph (volgen_graph_t *graph, dict_t *mod_dict)                  if (ret)                          goto out; +                ret = volgen_graph_set_iam_nfsd (&cgraph); +                if (ret) +                        goto out; +                  ret = volgen_graph_merge_sub (graph, &cgraph, 1);                  if (ret)                          goto out; diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c index e82339a537f..9d07b60973e 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c @@ -3008,6 +3008,31 @@ struct volopt_map_entry glusterd_volopt_map[] = {            .op_version  = GD_OP_VERSION_3_6_0,            .type        = NO_DOC,          }, +        { .key        = "cluster.halo-shd-max-latency", +          .voltype    = "cluster/replicate", +          .op_version = 2, +          .flags      = OPT_FLAG_CLIENT_OPT +        }, +        { .key        = "cluster.halo-nfsd-max-latency", +          .voltype    = "cluster/replicate", +          .op_version = 2, +          .flags      = OPT_FLAG_CLIENT_OPT +        }, +        { .key        = "cluster.halo-max-latency", +          .voltype    = "cluster/replicate", +          .op_version = 2, +          .flags      = OPT_FLAG_CLIENT_OPT +        }, +        { .key        = "cluster.halo-max-replicas", +          .voltype    = "cluster/replicate", +          .op_version = 2, +          .flags      = OPT_FLAG_CLIENT_OPT +        }, +        { .key        = "cluster.halo-min-replicas", +          .voltype    = "cluster/replicate", +          .op_version = 2, +          .flags      = OPT_FLAG_CLIENT_OPT +        },          { .key         = NULL          }  }; diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in index 6c4cdfed062..b5c90ba1dff 100755 --- a/xlators/mount/fuse/utils/mount.glusterfs.in +++ b/xlators/mount/fuse/utils/mount.glusterfs.in @@ -186,6 +186,21 @@ start_glusterfs ()      fi  #options with values start here +    if [ -n "$halo_max_latency" ]; then +      cmd_line=$(echo "$cmd_line --xlator-option \ + *replicate*.halo-max-latency=$halo_max_latency"); +    fi + +    if [ -n "$halo_max_replicas" ]; then +      cmd_line=$(echo "$cmd_line --xlator-option \ + *replicate*.halo-max-replicas=$halo_max_replicas"); +    fi + +    if [ -n "$halo_min_replicas" ]; then +      cmd_line=$(echo "$cmd_line --xlator-option \ + *replicate*.halo-min-replicas=$halo_min_replicas"); +    fi +      if [ -n "$log_level" ]; then          cmd_line=$(echo "$cmd_line --log-level=$log_level");      fi @@ -479,6 +494,15 @@ with_options()              [ -z "$fuse_mountopts" ] || fuse_mountopts="$fuse_mountopts,"              fuse_mountopts="${fuse_mountopts}$key=\"$value\""              ;; +        "halo-max-latency") +            halo_max_latency=$value +            ;; +        "halo-max-replicas") +            halo_max_replicas=$value +            ;; +        "halo-min-replicas") +          halo_min_replicas=$value +          ;;          x-*)              # comments or userspace application-specific options, drop them              ;; diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c index 3284facb893..944cae3a478 100644 --- a/xlators/protocol/client/src/client-handshake.c +++ b/xlators/protocol/client/src/client-handshake.c @@ -1548,7 +1548,7 @@ client_query_portmap_cbk (struct rpc_req *req, struct iovec *iov, int count, voi          rpc_clnt_reconfig (conf->rpc, &config);          conf->skip_notify = 1; -	conf->quick_reconnect = 1; +        conf->quick_reconnect = 1;  out:          if (frame) diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c index 3cb5e231fbe..0647f1035ba 100644 --- a/xlators/protocol/client/src/client.c +++ b/xlators/protocol/client/src/client.c @@ -467,7 +467,7 @@ int32_t  client_forget (xlator_t *this, inode_t *inode)  {          /* Nothing here */ -	return 0; +        return 0;  }  int32_t @@ -545,7 +545,7 @@ out:                  STACK_UNWIND_STRICT (lookup, frame, -1, ENOTCONN,                                       NULL, NULL, NULL, NULL); -	return 0; +        return 0;  } @@ -571,7 +571,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (stat, frame, -1, ENOTCONN, NULL, NULL); -	return 0; +        return 0;  } @@ -600,7 +600,7 @@ out:                  STACK_UNWIND_STRICT (truncate, frame, -1, ENOTCONN, NULL, NULL, NULL); -	return 0; +        return 0;  } @@ -628,7 +628,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOTCONN, NULL, NULL, NULL); -	return 0; +        return 0;  } @@ -657,7 +657,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (access, frame, -1, ENOTCONN, NULL); -	return 0; +        return 0;  } @@ -687,7 +687,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (readlink, frame, -1, ENOTCONN, NULL, NULL, NULL); -	return 0; +        return 0;  } @@ -718,7 +718,7 @@ out:                  STACK_UNWIND_STRICT (mknod, frame, -1, ENOTCONN,                                       NULL, NULL, NULL, NULL, NULL); -	return 0; +        return 0;  } @@ -748,7 +748,7 @@ out:                  STACK_UNWIND_STRICT (mkdir, frame, -1, ENOTCONN,                                       NULL, NULL, NULL, NULL, NULL); -	return 0; +        return 0;  } @@ -778,7 +778,7 @@ out:                  STACK_UNWIND_STRICT (unlink, frame, -1, ENOTCONN,                                       NULL, NULL, NULL); -	return 0; +        return 0;  }  int32_t @@ -807,7 +807,7 @@ out:                  STACK_UNWIND_STRICT (rmdir, frame, -1, ENOTCONN,                                       NULL, NULL, NULL); -	return 0; +        return 0;  } @@ -837,7 +837,7 @@ out:                  STACK_UNWIND_STRICT (symlink, frame, -1, ENOTCONN,                                       NULL, NULL, NULL, NULL, NULL); -	return 0; +        return 0;  } @@ -867,7 +867,7 @@ out:                  STACK_UNWIND_STRICT (rename, frame, -1, ENOTCONN,                                       NULL, NULL, NULL, NULL, NULL, NULL); -	return 0; +        return 0;  } @@ -897,7 +897,7 @@ out:                  STACK_UNWIND_STRICT (link, frame, -1, ENOTCONN,                                       NULL, NULL, NULL, NULL, NULL); -	return 0; +        return 0;  } @@ -932,7 +932,7 @@ out:                  STACK_UNWIND_STRICT (create, frame, -1, ENOTCONN,                                       NULL, NULL, NULL, NULL, NULL, NULL); -	return 0; +        return 0;  } @@ -965,7 +965,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (open, frame, -1, ENOTCONN, NULL, NULL); -	return 0; +        return 0;  } @@ -1000,7 +1000,7 @@ out:                  STACK_UNWIND_STRICT (readv, frame, -1, ENOTCONN,                                       NULL, 0, NULL, NULL, NULL); -	return 0; +        return 0;  } @@ -1038,7 +1038,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (writev, frame, -1, ENOTCONN, NULL, NULL, NULL); -	return 0; +        return 0;  } @@ -1064,7 +1064,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (flush, frame, -1, ENOTCONN, NULL); -	return 0; +        return 0;  } @@ -1093,7 +1093,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (fsync, frame, -1, ENOTCONN, NULL, NULL, NULL); -	return 0; +        return 0;  } @@ -1120,7 +1120,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (fstat, frame, -1, ENOTCONN, NULL, NULL); -	return 0; +        return 0;  } @@ -1149,7 +1149,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (opendir, frame, -1, ENOTCONN, NULL, NULL); -	return 0; +        return 0;  } @@ -1177,7 +1177,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOTCONN, NULL); -	return 0; +        return 0;  } @@ -1204,7 +1204,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (statfs, frame, -1, ENOTCONN, NULL, NULL); -	return 0; +        return 0;  }  static gf_boolean_t @@ -1393,7 +1393,7 @@ out:          if (need_unwind)                  STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL); -	return 0; +        return 0;  } @@ -1423,7 +1423,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOTCONN, NULL); -	return 0; +        return 0;  } @@ -1453,7 +1453,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOTCONN, NULL, NULL); -	return 0; +        return 0;  } @@ -1482,7 +1482,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (getxattr, frame, -1, ENOTCONN, NULL, NULL); -	return 0; +        return 0;  } @@ -1512,7 +1512,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (xattrop, frame, -1, ENOTCONN, NULL, NULL); -	return 0; +        return 0;  } @@ -1542,7 +1542,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOTCONN, NULL, NULL); -	return 0; +        return 0;  } @@ -1571,7 +1571,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (removexattr, frame, -1, ENOTCONN, NULL); -	return 0; +        return 0;  }  int32_t @@ -1598,7 +1598,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (fremovexattr, frame, -1, ENOTCONN, NULL); -	return 0; +        return 0;  }  int32_t @@ -1654,7 +1654,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (lk, frame, -1, ENOTCONN, NULL, NULL); -	return 0; +        return 0;  } @@ -1684,7 +1684,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (inodelk, frame, -1, ENOTCONN, NULL); -	return 0; +        return 0;  } @@ -1715,7 +1715,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (finodelk, frame, -1, ENOTCONN, NULL); -	return 0; +        return 0;  } @@ -1747,7 +1747,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (entrylk, frame, -1, ENOTCONN, NULL); -	return 0; +        return 0;  } @@ -1780,7 +1780,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOTCONN, NULL); -	return 0; +        return 0;  } @@ -1809,7 +1809,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOTCONN, 0, NULL, NULL); -	return 0; +        return 0;  }  int32_t @@ -1840,7 +1840,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (readdir, frame, -1, ENOTCONN, NULL, NULL); -	return 0; +        return 0;  } @@ -1872,7 +1872,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (readdirp, frame, -1, ENOTCONN, NULL, NULL); -	return 0; +        return 0;  } @@ -1901,7 +1901,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (setattr, frame, -1, ENOTCONN, NULL, NULL, NULL); -	return 0; +        return 0;  }  int32_t @@ -1929,7 +1929,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOTCONN, NULL, NULL, NULL); -	return 0; +        return 0;  }  int32_t @@ -2155,7 +2155,7 @@ out:          if (ret)                  STACK_UNWIND_STRICT (getspec, frame, -1, EINVAL, NULL); -	return 0; +        return 0;  } @@ -2227,6 +2227,15 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,          conf = this->private;          switch (event) { +        case RPC_CLNT_PING: +        { +                ret = default_notify (this, GF_EVENT_CHILD_PING, NULL); +                if (ret) +                        gf_log (this->name, GF_LOG_INFO, +                                "CHILD_PING notify failed"); +                conf->last_sent_event = GF_EVENT_CHILD_PING; +                break; +        }          case RPC_CLNT_CONNECT:          {                  conf->connected = 1; @@ -2318,7 +2327,6 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,                  } else {                          rpc->conn.config.remote_port = 0; -                  }                  break; @@ -2670,7 +2678,7 @@ reconfigure (xlator_t *this, dict_t *options)          ret = 0;  out: -	return ret; +        return ret;  }  | 
