gNFS: RFE for NFS connection behavior

Implement reconfigure() for NFS xlator so that volume set/reset wont
restart the NFS server process. But few options can not be reconfigured
dynamically e.g. nfs.mem-factor, nfs.port etc which needs NFS to be
restarted.

Change-Id: Ic586fd55b7933c0a3175708d8c41ed0475d74a1c
BUG: 1027409
Signed-off-by: Santosh Kumar Pradhan <spradhan@redhat.com>
Reviewed-on: http://review.gluster.org/6236
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Rajesh Joseph <rjoseph@redhat.com>
Reviewed-by: Anand Avati <avati@redhat.com>

7 files changed, 269 insertions, 39 deletions

diff --git a/rpc/rpc-lib/src/rpc-drc.c b/rpc/rpc-lib/src/rpc-drc.c
index 66d07cfe6..8181e6aee 100644
--- a/rpc/rpc-lib/src/rpc-drc.c
+++ b/rpc/rpc-lib/src/rpc-drc.c
@@ -708,10 +708,15 @@ rpcsvc_drc_init (rpcsvc_t *svc, dict_t *options)
         uint32_t                    drc_size       = 0;
         uint32_t                    drc_factor     = 0;
         rpcsvc_drc_globals_t       *drc            = NULL;
+        static gf_boolean_t         drc_inited     = _gf_false;
 
         GF_ASSERT (svc);
         GF_ASSERT (options);
 
+        /* Already inited */
+        if (drc_inited)
+                return 0;
+
         if (!svc->drc) {
                 drc = GF_CALLOC (1, sizeof (rpcsvc_drc_globals_t),
                                  gf_common_mt_drc_globals_t);
@@ -732,11 +737,12 @@ rpcsvc_drc_init (rpcsvc_t *svc, dict_t *options)
 
         /* Toggle DRC on/off, when more drc types(persistent/cluster)
            are added, we shouldn't treat this as boolean */
-        ret = dict_get_str_boolean (options, "nfs.drc", _gf_false);
+        ret = dict_get_str_boolean (options, "nfs.drc", _gf_true);
         if (ret == -1) {
                 gf_log (GF_RPCSVC, GF_LOG_INFO, "drc user options need second look");
                 ret = _gf_true;
         }
+        drc->enable_drc = ret;
 
         if (ret == _gf_false) {
                 /* drc off */
@@ -796,6 +802,7 @@ rpcsvc_drc_init (rpcsvc_t *svc, dict_t *options)
 
         gf_log (GF_RPCSVC, GF_LOG_DEBUG, "drc init successful");
         drc->status = DRC_INITIATED;
+        drc_inited = _gf_true;
 
  out:
         UNLOCK (&drc->lock);
@@ -809,3 +816,57 @@ rpcsvc_drc_init (rpcsvc_t *svc, dict_t *options)
         }
         return ret;
 }
+
+int
+rpcsvc_drc_reconfigure (rpcsvc_t *svc, dict_t *options)
+{
+        int                     ret        = -1;
+        gf_boolean_t            enable_drc = _gf_false;
+        rpcsvc_drc_globals_t    *drc       = NULL;
+        uint32_t                drc_size   = 0;
+
+        if ((!svc) || (!options))
+                return (-1);
+
+        drc = svc->drc;
+        /* reconfig for drc-size */
+        if (dict_get_uint32 (options, "nfs.drc-size", &drc_size))
+                drc_size = DRC_DEFAULT_CACHE_SIZE;
+
+        if (drc->global_cache_size != drc_size) {
+                gf_log (GF_RPCSVC, GF_LOG_DEBUG, "nfs.drc-size size can not "
+                        "be reconfigured without NFS server restart.");
+                return (-1);
+        }
+
+        /* reconfig for nfs.drc */
+        ret = dict_get_str_boolean (options, "nfs.drc", _gf_true);
+        if (ret < 0) {
+                ret = _gf_true;
+        }
+        enable_drc = ret;
+
+        if (drc->enable_drc == enable_drc)
+                return 0;
+
+        drc->enable_drc = enable_drc;
+        if (enable_drc) {
+                if (drc == NULL)
+                        return rpcsvc_drc_init(svc, options);
+        } else {
+                if (drc == NULL)
+                        return (0);
+
+                LOCK (&drc->lock);
+                (void) rpcsvc_unregister_notify (svc, rpcsvc_drc_notify, THIS);
+                if (drc->mempool) {
+                        mem_pool_destroy (drc->mempool);
+                        drc->mempool = NULL;
+                }
+                UNLOCK (&drc->lock);
+                GF_FREE (drc);
+                svc->drc = NULL;
+        }
+
+        return (0);
+}
diff --git a/rpc/rpc-lib/src/rpc-drc.h b/rpc/rpc-lib/src/rpc-drc.h
index 0a1688992..7dfaef978 100644
--- a/rpc/rpc-lib/src/rpc-drc.h
+++ b/rpc/rpc-lib/src/rpc-drc.h
@@ -71,6 +71,7 @@ struct drc_globals {
         struct list_head          cache_head;
         uint32_t                  client_count;
         struct list_head          clients_head;
+        gf_boolean_t              enable_drc;
 };
 
 int
@@ -97,4 +98,7 @@ rpcsvc_drc_priv (rpcsvc_drc_globals_t *drc);
 int
 rpcsvc_drc_init (rpcsvc_t *svc, dict_t *options);
 
+int
+rpcsvc_drc_reconfigure (rpcsvc_t *svc, dict_t *options);
+
 #endif /* RPC_DRC_H */
diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c
index 5f2e91c70..c24d41084 100644
--- a/rpc/rpc-lib/src/rpc-transport.c
+++ b/rpc/rpc-lib/src/rpc-transport.c
@@ -295,7 +295,7 @@ rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *trans_name)
 	}
 
         *VOID(&(trans->reconfigure)) = dlsym (handle, "reconfigure");
-        if (trans->fini == NULL) {
+        if (trans->reconfigure == NULL) {
                 gf_log ("rpc-transport", GF_LOG_DEBUG,
                         "dlsym (gf_rpc_transport_reconfigure) on %s", dlerror());
         }
diff --git a/rpc/rpc-lib/src/rpcsvc-auth.c b/rpc/rpc-lib/src/rpcsvc-auth.c
index 7ee96d85b..4cb86a758 100644
--- a/rpc/rpc-lib/src/rpcsvc-auth.c
+++ b/rpc/rpc-lib/src/rpcsvc-auth.c
@@ -178,6 +178,29 @@ err:
 }
 
 int
+rpcsvc_set_addr_namelookup (rpcsvc_t *svc, dict_t *options)
+{
+        int             ret;
+        static char     *addrlookup_key = "rpc-auth.addr.namelookup";
+
+        if (!svc || !options)
+                return (-1);
+
+        /* By default it's disabled */
+        ret = dict_get_str_boolean (options, addrlookup_key, _gf_false);
+        if (ret < 0) {
+                svc->addr_namelookup = _gf_false;
+        } else {
+                svc->addr_namelookup = ret;
+        }
+
+        if (svc->addr_namelookup)
+                gf_log (GF_RPCSVC, GF_LOG_DEBUG, "Addr-Name lookup enabled");
+
+        return (0);
+}
+
+int
 rpcsvc_set_allow_insecure (rpcsvc_t *svc, dict_t *options)
 {
         int             ret = -1;
@@ -233,6 +256,7 @@ rpcsvc_auth_init (rpcsvc_t *svc, dict_t *options)
 
         (void) rpcsvc_set_allow_insecure (svc, options);
         (void) rpcsvc_set_root_squash (svc, options);
+        (void) rpcsvc_set_addr_namelookup (svc, options);
         ret = rpcsvc_auth_add_initers (svc);
         if (ret == -1) {
                 gf_log (GF_RPCSVC, GF_LOG_ERROR, "Failed to add initers");
@@ -249,6 +273,25 @@ out:
         return ret;
 }
 
+int
+rpcsvc_auth_reconf (rpcsvc_t *svc, dict_t *options)
+{
+        int ret = 0;
+
+        if ((!svc) || (!options))
+                return (-1);
+
+        ret = rpcsvc_set_allow_insecure (svc, options);
+        if (ret)
+                return (-1);
+
+        ret = rpcsvc_set_root_squash (svc, options);
+        if (ret)
+                return (-1);
+
+        return rpcsvc_set_addr_namelookup (svc, options);
+}
+
 
 rpcsvc_auth_t *
 __rpcsvc_auth_get_handler (rpcsvc_request_t *req)
diff --git a/rpc/rpc-lib/src/rpcsvc-common.h b/rpc/rpc-lib/src/rpcsvc-common.h
index 53c1a8fe3..aed55e039 100644
--- a/rpc/rpc-lib/src/rpcsvc-common.h
+++ b/rpc/rpc-lib/src/rpcsvc-common.h
@@ -52,28 +52,29 @@ typedef struct rpcsvc_state {
         dict_t                  *options;
 
         /* Allow insecure ports. */
-        int                     allow_insecure;
+        gf_boolean_t            allow_insecure;
         gf_boolean_t            register_portmap;
         gf_boolean_t            root_squash;
         glusterfs_ctx_t         *ctx;
 
         /* list of connections which will listen for incoming connections */
-        struct list_head         listeners;
+        struct list_head        listeners;
 
         /* list of programs registered with rpcsvc */
-        struct list_head         programs;
+        struct list_head        programs;
 
         /* list of notification callbacks */
-        struct list_head         notify;
-        int                      notify_count;
+        struct list_head        notify;
+        int                     notify_count;
 
         void                    *mydata; /* This is xlator */
-        rpcsvc_notify_t          notifyfn;
+        rpcsvc_notify_t         notifyfn;
         struct mem_pool         *rxpool;
         rpcsvc_drc_globals_t    *drc;
 
 	/* per-client limit of outstanding rpc requests */
-	int                      outstanding_rpc_limit;
+        int                     outstanding_rpc_limit;
+        gf_boolean_t            addr_namelookup;
 } rpcsvc_t;
 
 /* DRC START */
diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c
index 8fe2e52bc..037c157f2 100644
--- a/rpc/rpc-lib/src/rpcsvc.c
+++ b/rpc/rpc-lib/src/rpcsvc.c
@@ -1249,12 +1249,13 @@ rpcsvc_error_reply (rpcsvc_request_t *req)
 inline int
 rpcsvc_program_register_portmap (rpcsvc_program_t *newprog, uint32_t port)
 {
-        int                ret   = 0;
+        int                ret   = -1; /* FAIL */
 
         if (!newprog) {
                 goto out;
         }
 
+        /* pmap_set() returns 0 for FAIL and 1 for SUCCESS */
         if (!(pmap_set (newprog->prognum, newprog->progver, IPPROTO_TCP,
                         port))) {
                 gf_log (GF_RPCSVC, GF_LOG_ERROR, "Could not register with"
@@ -1262,16 +1263,16 @@ rpcsvc_program_register_portmap (rpcsvc_program_t *newprog, uint32_t port)
                 goto out;
         }
 
-        ret = 0;
+        ret = 0; /* SUCCESS */
 out:
         return ret;
 }
 
 
-static inline int
+inline int
 rpcsvc_program_unregister_portmap (rpcsvc_program_t *prog)
 {
-        int ret = 0;
+        int ret = -1;
 
         if (!prog)
                 goto out;
@@ -1889,27 +1890,89 @@ rpcsvc_init_options (rpcsvc_t *svc, dict_t *options)
                 gf_log (GF_RPCSVC, GF_LOG_DEBUG, "Portmap registration "
                         "disabled");
 
-        svc->outstanding_rpc_limit = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT;
+        ret = rpcsvc_set_outstanding_rpc_limit (svc, options);
+out:
+        return ret;
+}
 
-        if (dict_get (options, "rpc.outstanding-rpc-limit")) {
-                ret = dict_get_str (options, "rpc.oustanding-rpc-limit",
-                                    &optstr);
-                if (ret < 0) {
-                        gf_log (GF_RPCSVC, GF_LOG_ERROR, "Value went missing");
-                        goto out;
+int
+rpcsvc_reconfigure_options (rpcsvc_t *svc, dict_t *options)
+{
+        xlator_t         *xlator    = NULL;
+        xlator_list_t    *volentry  = NULL;
+        char             *srchkey   = NULL;
+        char             *keyval    = NULL;
+        int              ret        = -1;
+
+        if ((!svc) || (!svc->options) || (!options))
+                return (-1);
+
+        /* Fetch the xlator from svc */
+        xlator = (xlator_t *) svc->mydata;
+        if (!xlator)
+                return (-1);
+
+        /* Reconfigure the volume specific rpc-auth.addr allow part */
+        volentry = xlator->children;
+        while (volentry) {
+                ret = gf_asprintf (&srchkey, "rpc-auth.addr.%s.allow",
+                                             volentry->xlator->name);
+                if (ret == -1) {
+                        gf_log (GF_RPCSVC, GF_LOG_ERROR, "asprintf failed");
+                        return (-1);
                 }
 
-                ret = gf_string2int32 (optstr, &svc->outstanding_rpc_limit);
-                if (ret < 0) {
-                        gf_log (GF_RPCSVC, GF_LOG_ERROR, "Invalid RPC limit %s",
-                                optstr);
-                        goto out;
+                /* If found the srchkey, delete old key/val pair
+                 * and set the key with new value.
+                 */
+                if (!dict_get_str (options, srchkey, &keyval)) {
+                        dict_del (svc->options, srchkey);
+                        ret = dict_set_str (svc->options, srchkey, keyval);
+                        if (ret < 0) {
+                                gf_log (GF_RPCSVC, GF_LOG_ERROR,
+                                        "dict_set_str error");
+                                GF_FREE (srchkey);
+                                return (-1);
+                        }
                 }
+
+                GF_FREE (srchkey);
+                volentry = volentry->next;
         }
 
-        ret = 0;
-out:
-        return ret;
+        /* Reconfigure the volume specific rpc-auth.addr reject part */
+        volentry = xlator->children;
+        while (volentry) {
+                ret = gf_asprintf (&srchkey, "rpc-auth.addr.%s.reject",
+                                             volentry->xlator->name);
+                if (ret == -1) {
+                        gf_log (GF_RPCSVC, GF_LOG_ERROR, "asprintf failed");
+                        return (-1);
+                }
+
+                /* If found the srchkey, delete old key/val pair
+                 * and set the key with new value.
+                 */
+                if (!dict_get_str (options, srchkey, &keyval)) {
+                        dict_del (svc->options, srchkey);
+                        ret = dict_set_str (svc->options, srchkey, keyval);
+                        if (ret < 0) {
+                                gf_log (GF_RPCSVC, GF_LOG_ERROR,
+                                        "dict_set_str error");
+                                GF_FREE (srchkey);
+                                return (-1);
+                        }
+                }
+
+                GF_FREE (srchkey);
+                volentry = volentry->next;
+        }
+
+        ret = rpcsvc_init_options (svc, options);
+        if (ret)
+                return (-1);
+
+        return rpcsvc_auth_reconf (svc, options);
 }
 
 int
@@ -1958,6 +2021,48 @@ out:
         return ret;
 }
 
+/*
+ * Reconfigure() the rpc.outstanding-rpc-limit param.
+ */
+int
+rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options)
+{
+        int            ret        = -1; /* FAILURE */
+        int            rpclim     = 0;
+        static char    *rpclimkey = "rpc.outstanding-rpc-limit";
+
+        if ((!svc) || (!options))
+                return (-1);
+
+        /* Reconfigure() the rpc.outstanding-rpc-limit param */
+        ret = dict_get_int32 (options, rpclimkey, &rpclim);
+        if (ret < 0) {
+                /* Fall back to default for FAILURE */
+                rpclim = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT;
+        } else {
+                /* SUCCESS: round off to multiple of 8.
+                 * If the input value fails Boundary check, fall back to
+                 * default i.e. RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT.
+                 * NB: value 0 is special, means its unset i.e. unlimited.
+                 */
+                rpclim = ((rpclim + 8 - 1) >> 3) * 8;
+                if (rpclim < RPCSVC_MIN_OUTSTANDING_RPC_LIMIT) {
+                        rpclim = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT;
+                } else if (rpclim > RPCSVC_MAX_OUTSTANDING_RPC_LIMIT) {
+                        rpclim = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT;
+                }
+        }
+
+        if (svc->outstanding_rpc_limit != rpclim) {
+                svc->outstanding_rpc_limit = rpclim;
+                gf_log (GF_RPCSVC, GF_LOG_INFO,
+                                   "Configured %s with value %d",
+                                   rpclimkey, rpclim);
+        }
+
+        return (0);
+}
+
 /* The global RPC service initializer.
  */
 rpcsvc_t *
@@ -2089,7 +2194,7 @@ err:
 }
 
 
-int
+static int
 rpcsvc_transport_peer_check_allow (dict_t *options, char *volname,
                                    char *ip, char *hostname)
 {
@@ -2118,7 +2223,7 @@ out:
         return ret;
 }
 
-int
+static int
 rpcsvc_transport_peer_check_reject (dict_t *options, char *volname,
                                     char *ip, char *hostname)
 {
@@ -2170,7 +2275,7 @@ rpcsvc_combine_allow_reject_volume_check (int allow, int reject)
 }
 
 int
-rpcsvc_auth_check (dict_t *options, char *volname,
+rpcsvc_auth_check (rpcsvc_t *svc, char *volname,
                    rpc_transport_t *trans)
 {
         int     ret                            = RPCSVC_AUTH_REJECT;
@@ -2182,8 +2287,14 @@ rpcsvc_auth_check (dict_t *options, char *volname,
         char   *allow_str                      = NULL;
         char   *reject_str                     = NULL;
         char   *srchstr                        = NULL;
+        dict_t *options                        = NULL;
+
+        if (!svc || !volname || !trans)
+                return ret;
 
-        if (!options || !volname || !trans)
+        /* Fetch the options from svc struct and validate */
+        options = svc->options;
+        if (!options)
                 return ret;
 
         ret = rpcsvc_transport_peername (trans, client_ip, RPCSVC_PEER_STRLEN);
@@ -2225,9 +2336,8 @@ rpcsvc_auth_check (dict_t *options, char *volname,
         if (!get_host_name (client_ip, &ip))
                 ip = client_ip;
 
-        /* addr-namelookup disabled by default */
-        ret = dict_get_str_boolean (options, "rpc-auth.addr.namelookup", 0);
-        if (ret == _gf_true) {
+        /* addr-namelookup check */
+        if (svc->addr_namelookup == _gf_true) {
                 ret = gf_get_hostname_from_ip (ip, &hostname);
                 if (ret) {
                         if (hostname)
diff --git a/rpc/rpc-lib/src/rpcsvc.h b/rpc/rpc-lib/src/rpcsvc.h
index ac2f09bea..cbc1f4226 100644
--- a/rpc/rpc-lib/src/rpcsvc.h
+++ b/rpc/rpc-lib/src/rpcsvc.h
@@ -40,6 +40,7 @@
 
 #define RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT 64
 #define RPCSVC_MAX_OUTSTANDING_RPC_LIMIT 65536
+#define RPCSVC_MIN_OUTSTANDING_RPC_LIMIT 0 /* No limit i.e. Unlimited */
 
 #define GF_RPCSVC       "rpc-service"
 #define RPCSVC_THREAD_STACK_SIZE ((size_t)(1024 * GF_UNIT_KB))
@@ -440,6 +441,9 @@ extern int
 rpcsvc_program_register_portmap (rpcsvc_program_t *newprog, uint32_t port);
 
 extern int
+rpcsvc_program_unregister_portmap (rpcsvc_program_t *newprog);
+
+extern int
 rpcsvc_register_portmap_enabled (rpcsvc_t *svc);
 
 /* Inits the global RPC service data structures.
@@ -449,6 +453,9 @@ extern rpcsvc_t *
 rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options,
              uint32_t poolcount);
 
+extern int
+rpcsvc_reconfigure_options (rpcsvc_t *svc, dict_t *options);
+
 int
 rpcsvc_register_notify (rpcsvc_t *svc, rpcsvc_notify_t notify, void *mydata);
 
@@ -491,8 +498,7 @@ rpcsvc_transport_peeraddr (rpc_transport_t *trans, char *addrstr, int addrlen,
                            struct sockaddr_storage *returnsa, socklen_t sasize);
 
 extern int
-rpcsvc_auth_check (dict_t *options, char *volname,
-                   rpc_transport_t *trans);
+rpcsvc_auth_check (rpcsvc_t *svc, char *volname, rpc_transport_t *trans);
 
 extern int
 rpcsvc_transport_privport_check (rpcsvc_t *svc, char *volname,
@@ -553,6 +559,9 @@ extern int
 rpcsvc_auth_init (rpcsvc_t *svc, dict_t *options);
 
 extern int
+rpcsvc_auth_reconf (rpcsvc_t *svc, dict_t *options);
+
+extern int
 rpcsvc_auth_transport_init (rpc_transport_t *xprt);
 
 extern int
@@ -584,11 +593,13 @@ rpcsvc_transport_unix_options_build (dict_t **options, char *filepath);
 int
 rpcsvc_set_allow_insecure (rpcsvc_t *svc, dict_t *options);
 int
+rpcsvc_set_addr_namelookup (rpcsvc_t *svc, dict_t *options);
+int
 rpcsvc_set_root_squash (rpcsvc_t *svc, dict_t *options);
 int
+rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options);
+int
 rpcsvc_auth_array (rpcsvc_t *svc, char *volname, int *autharr, int arrlen);
-char *
-rpcsvc_volume_allowed (dict_t *options, char *volname);
 rpcsvc_vector_sizer
 rpcsvc_get_program_vector_sizer (rpcsvc_t *svc, uint32_t prognum,
                                  uint32_t progver, uint32_t procnum);