glusterd-volgen: Improve volume op-versions calculation

Backport of patch on master branch, under review at
        http://review.gluster.org/4952

Volume op-versions calculations now take into account if an option,
 a. enables/disables an xlator, or
 b. is a boolean option.

This prevents op-versions from being updated when a feature is disabled.

BUG: 954256
Change-Id: Ic68032b9e55a3f0191f8fc3ecd6b5ced385ad943
Signed-off-by: Kaushal M <kaushal@redhat.com>
Reviewed-on: http://review.gluster.org/5094
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

7 files changed, 591 insertions, 480 deletions

diff --git a/libglusterfs/src/options.c b/libglusterfs/src/options.c
index 5d436bab38d..842b6413ab4 100644
--- a/libglusterfs/src/options.c
+++ b/libglusterfs/src/options.c
@@ -761,7 +761,7 @@ out:
 }
 
 
-static volume_option_t *
+volume_option_t *
 xlator_volume_option_get_list (volume_opt_list_t *vol_list, const char *key)
 {
         volume_option_t         *opt = NULL;
@@ -1098,7 +1098,6 @@ pc_or_size (char *in, double *out)
         return ret;
 }
 
-
 DEFINE_INIT_OPT(char *, str, pass);
 DEFINE_INIT_OPT(uint64_t, uint64, gf_string2uint64);
 DEFINE_INIT_OPT(int64_t, int64, gf_string2int64);
diff --git a/libglusterfs/src/options.h b/libglusterfs/src/options.h
index 64c2c64ef59..e2a25baa9e7 100644
--- a/libglusterfs/src/options.h
+++ b/libglusterfs/src/options.h
@@ -93,6 +93,9 @@ int xlator_options_validate (xlator_t *xl, dict_t *options, char **errstr);
 volume_option_t *
 xlator_volume_option_get (xlator_t *xl, const char *key);
 
+volume_option_t *
+xlator_volume_option_get_list (volume_opt_list_t *vol_list, const char *key);
+
 
 #define DECLARE_INIT_OPT(type_t, type)                                  \
 int                                                                     \
diff --git a/libglusterfs/src/xlator.c b/libglusterfs/src/xlator.c
index 3fb6eeebb09..7f29d180a2d 100644
--- a/libglusterfs/src/xlator.c
+++ b/libglusterfs/src/xlator.c
@@ -119,18 +119,9 @@ xlator_volopt_dynload (char *xlator_type, void **dl_handle,
         int                     ret = -1;
         char                    *name = NULL;
         void                    *handle = NULL;
-        volume_opt_list_t       *vol_opt = NULL;
 
         GF_VALIDATE_OR_GOTO ("xlator", xlator_type, out);
 
-        GF_ASSERT (dl_handle);
-
-        if (*dl_handle)
-                if (dlclose (*dl_handle))
-                        gf_log ("xlator", GF_LOG_WARNING, "Unable to close "
-                                  "previously opened handle( may be stale)."
-                                  "Ignoring the invalid handle");
-
         ret = gf_asprintf (&name, "%s/%s.so", XLATORDIR, xlator_type);
         if (-1 == ret) {
                 gf_log ("xlator", GF_LOG_ERROR, "asprintf failed");
@@ -146,25 +137,15 @@ xlator_volopt_dynload (char *xlator_type, void **dl_handle,
                 gf_log ("xlator", GF_LOG_WARNING, "%s", dlerror ());
                 goto out;
         }
-        *dl_handle = handle;
-
 
-        vol_opt = GF_CALLOC (1, sizeof (volume_opt_list_t),
-                         gf_common_mt_volume_opt_list_t);
-
-        if (!vol_opt) {
-                goto out;
-        }
-
-        if (!(vol_opt->given_opt = dlsym (handle, "options"))) {
+        if (!(opt_list->given_opt = dlsym (handle, "options"))) {
                 dlerror ();
-                gf_log ("xlator", GF_LOG_DEBUG,
-                         "Strict option validation not enforced -- neglecting");
+                gf_log ("xlator", GF_LOG_ERROR,
+                        "Failed to load xlator opt table");
+                goto out;
         }
-        opt_list->given_opt = vol_opt->given_opt;
 
-        INIT_LIST_HEAD (&vol_opt->list);
-        list_add_tail (&vol_opt->list, &opt_list->list);
+        *dl_handle = handle;
 
         ret = 0;
  out:
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 0309a539cbe..6cb69615e3d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -7497,12 +7497,23 @@ _update_volume_op_versions (dict_t *this, char *key, data_t *value, void *data)
 {
         int                op_version = 0;
         glusterd_volinfo_t *ctx       = NULL;
+        gf_boolean_t       enabled    = _gf_true;
+        int                ret        = -1;
 
         GF_ASSERT (data);
         ctx = data;
 
         op_version = glusterd_get_op_version_for_key (key);
 
+        if (gd_is_xlator_option (key) || gd_is_boolean_option (key)) {
+                ret = gf_string2boolean (value->data, &enabled);
+                if (ret)
+                        return 0;
+
+                if (!enabled)
+                        return 0;
+        }
+
         if (op_version > ctx->op_version)
                 ctx->op_version = op_version;
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 3d07412b207..c6a78ea15ec 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -15,6 +15,7 @@
 
 #include <fnmatch.h>
 #include <sys/wait.h>
+#include <dlfcn.h>
 
 #if (HAVE_LIB_XML)
 #include <libxml/encoding.h>
@@ -34,6 +35,7 @@
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "run.h"
+#include "options.h"
 
 extern struct volopt_map_entry glusterd_volopt_map[];
 
@@ -1863,7 +1865,7 @@ xml_add_volset_element (xmlTextWriterPtr writer, const char *name,
 #endif
 
 static int
-get_key_from_volopt ( struct volopt_map_entry *vme, char **key)
+_get_xlator_opt_key_from_vme ( struct volopt_map_entry *vme, char **key)
 {
         int ret = 0;
 
@@ -1906,6 +1908,19 @@ get_key_from_volopt ( struct volopt_map_entry *vme, char **key)
         return ret;
 }
 
+static void
+_free_xlator_opt_key (char *key)
+{
+        GF_ASSERT (key);
+
+        if (!strcmp (key, AUTH_ALLOW_OPT_KEY) ||
+            !strcmp (key, AUTH_REJECT_OPT_KEY) ||
+            !strcmp (key, NFS_DISABLE_OPT_KEY))
+                GF_FREE (key);
+
+        return;
+}
+
 int
 glusterd_get_volopt_content (dict_t * ctx, gf_boolean_t xml_out)
 {
@@ -1936,36 +1951,40 @@ glusterd_get_volopt_content (dict_t * ctx, gf_boolean_t xml_out)
 
         for (vme = &glusterd_volopt_map[0]; vme->key; vme++) {
 
-                if ( ( vme->type == NO_DOC) || (vme->type == GLOBAL_NO_DOC) )
+                if ((vme->type == NO_DOC) || (vme->type == GLOBAL_NO_DOC))
                         continue;
 
-                if (get_key_from_volopt (vme, &key))
-                                goto out; /*Some error while getin key*/
-
                 if (vme->description) {
                         descr = vme->description;
                         def_val = vme->value;
                 } else {
+                        if (_get_xlator_opt_key_from_vme (vme, &key))
+                                goto out; /*Some error while getting key*/
+
                         if (!xlator_type || strcmp (vme->voltype, xlator_type)){
-                               ret = xlator_volopt_dynload (vme->voltype,
-                                                            &dl_handle,
-                                                            &vol_opt_handle);
+                                ret = xlator_volopt_dynload (vme->voltype,
+                                                             &dl_handle,
+                                                             &vol_opt_handle);
                                 if (ret) {
-                                        dl_handle = NULL;
-                                        continue;
+                                        ret = 0;
+                                        goto cont;
                                 }
                         }
                         ret = xlator_option_info_list (&vol_opt_handle, key,
                                                        &def_val, &descr);
-                        if (ret) /*Swallow Error i.e if option not found*/
-                                continue;
+                        if (ret) {/*Swallow Error i.e if option not found*/
+                                ret = 0;
+                                goto cont;
+                        }
                 }
 
                 if (xml_out) {
 #if (HAVE_LIB_XML)
                         if (xml_add_volset_element (writer,vme->key,
-                                                    def_val, descr))
-                                goto out;
+                                                    def_val, descr)) {
+                                ret = -1;
+                                goto cont;
+                        }
 #else
                         gf_log ("glusterd", GF_LOG_ERROR, "Libxml not present");
 #endif
@@ -1975,11 +1994,18 @@ glusterd_get_volopt_content (dict_t * ctx, gf_boolean_t xml_out)
                                   vme->key, def_val, descr);
                         strcat (output_string, tmp_str);
                 }
-
-                if (!strcmp (key, AUTH_ALLOW_OPT_KEY) ||
-                    !strcmp (key, AUTH_REJECT_OPT_KEY) ||
-                    !strcmp (key, NFS_DISABLE_OPT_KEY))
-                        GF_FREE (key);
+cont:
+                if (dl_handle) {
+                        dlclose (dl_handle);
+                        dl_handle = NULL;
+                        vol_opt_handle.given_opt = NULL;
+                }
+                if (key) {
+                        _free_xlator_opt_key (key);
+                        key = NULL;
+                }
+                if (ret)
+                        goto out;
         }
 
 #if (HAVE_LIB_XML)
@@ -2006,7 +2032,7 @@ glusterd_get_volopt_content (dict_t * ctx, gf_boolean_t xml_out)
         }
 
         ret = dict_set_dynstr (ctx, "help-str", output);
- out:
+out:
         gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 
@@ -3693,36 +3719,111 @@ out:
         return ret;
 }
 
-uint32_t
-glusterd_get_op_version_for_key (char *key)
-{
+static struct volopt_map_entry *
+_gd_get_vmep (char *key) {
         char *completion = NULL;
         struct volopt_map_entry *vmep = NULL;
-        int   ret = 0;
+        int ret = 0;
 
-        COMPLETE_OPTION(key, completion, ret);
+        COMPLETE_OPTION ((char *)key, completion, ret);
         for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
-                if (strcmp (vmep->key, key) == 0) {
-                        return vmep->op_version;
-                }
+                if (strcmp (vmep->key, key) == 0)
+                        return vmep;
         }
 
+        return NULL;
+}
+
+uint32_t
+glusterd_get_op_version_for_key (char *key)
+{
+        struct volopt_map_entry *vmep = NULL;
+
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+        if (vmep)
+                return vmep->op_version;
+
         return 0;
 }
 
 gf_boolean_t
 gd_is_client_option (char *key)
 {
-        char *completion = NULL;
         struct volopt_map_entry *vmep = NULL;
-        int   ret = 0;
 
-        COMPLETE_OPTION(key, completion, ret);
-        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
-                if (strcmp (vmep->key, key) == 0) {
-                        return vmep->client_option;
-                }
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+        if (vmep && (vmep->flags & OPT_FLAG_CLIENT_OPT))
+                return _gf_true;
+
+        return _gf_false;
+}
+
+gf_boolean_t
+gd_is_xlator_option (char *key)
+{
+        struct volopt_map_entry *vmep = NULL;
+
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+        if (vmep && (vmep->flags & OPT_FLAG_XLATOR_OPT))
+                return _gf_true;
+
+        return _gf_false;
+}
+
+volume_option_type_t
+_gd_get_option_type (char *key)
+{
+        struct volopt_map_entry *vmep = NULL;
+        void                    *dl_handle = NULL;
+        volume_opt_list_t       vol_opt_list = {{0},};
+        int                     ret = -1;
+        volume_option_t         *opt = NULL;
+        char                    *xlopt_key = NULL;
+        volume_option_type_t    opt_type = GF_OPTION_TYPE_MAX;
+
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+
+        if (vmep) {
+                INIT_LIST_HEAD (&vol_opt_list.list);
+                ret = xlator_volopt_dynload (vmep->voltype, &dl_handle,
+                                             &vol_opt_list);
+                if (ret)
+                        goto out;
+
+                if (_get_xlator_opt_key_from_vme (vmep, &xlopt_key))
+                        goto out;
+
+                opt = xlator_volume_option_get_list (&vol_opt_list, xlopt_key);
+                _free_xlator_opt_key (xlopt_key);
+
+                if (opt)
+                        opt_type = opt->type;
         }
 
+out:
+        if (dl_handle) {
+                dlclose (dl_handle);
+                dl_handle = NULL;
+        }
+
+        return opt_type;
+}
+
+gf_boolean_t
+gd_is_boolean_option (char *key)
+{
+        GF_ASSERT (key);
+
+        if (GF_OPTION_TYPE_BOOL == _gd_get_option_type (key))
+                return _gf_true;
+
         return _gf_false;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
index 746c6e92bc8..6a41f47c16c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -60,7 +60,9 @@ typedef enum {
 
 typedef enum gd_volopt_flags_ {
         OPT_FLAG_NONE,
-        OPT_FLAG_FORCE = 1,
+        OPT_FLAG_FORCE = 0x01,      // option needs force to be reset
+        OPT_FLAG_XLATOR_OPT = 0x02, // option enables/disables xlators
+        OPT_FLAG_CLIENT_OPT = 0x04, // option affects clients
 } gd_volopt_flags_t;
 
 typedef enum {
@@ -102,7 +104,7 @@ struct volopt_map_entry {
         /* If client_option is true, the option affects clients.
          * this is used to calculate client-op-version of volumes
          */
-        gf_boolean_t client_option;
+        //gf_boolean_t client_option;
 };
 
 int glusterd_create_rb_volfiles (glusterd_volinfo_t *volinfo,
@@ -143,4 +145,11 @@ glusterd_get_op_version_for_key (char *key);
 
 gf_boolean_t
 gd_is_client_option (char *key);
+
+gf_boolean_t
+gd_is_xlator_option (char *key);
+
+gf_boolean_t
+gd_is_boolean_option (char *key);
+
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index fa2df0338e4..cca1f9f49c6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -347,179 +347,179 @@ out:
 
 struct volopt_map_entry glusterd_volopt_map[] = {
         /* DHT xlator options */
-        { .key           = "cluster.lookup-unhashed",
-          .voltype       = "cluster/distribute",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.min-free-disk",
-          .voltype       = "cluster/distribute",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.min-free-inodes",
-          .voltype       = "cluster/distribute",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.rebalance-stats",
-          .voltype       = "cluster/distribute",
-          .op_version    = 2,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.subvols-per-directory",
-          .voltype       = "cluster/distribute",
-          .option        = "directory-layout-spread",
-          .op_version    = 2,
-          .validate_fn   = validate_subvols_per_directory,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.readdir-optimize",
-          .voltype       = "cluster/distribute",
-          .op_version    = 2,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.nufa",
-          .voltype       = "cluster/distribute",
-          .option        = "!nufa",
-          .type          = NO_DOC,
-          .op_version    = 2,
-          .client_option = _gf_true
+        { .key        = "cluster.lookup-unhashed",
+          .voltype    = "cluster/distribute",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.min-free-disk",
+          .voltype    = "cluster/distribute",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.min-free-inodes",
+          .voltype    = "cluster/distribute",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.rebalance-stats",
+          .voltype    = "cluster/distribute",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "cluster.subvols-per-directory",
+          .voltype     = "cluster/distribute",
+          .option      = "directory-layout-spread",
+          .op_version  = 2,
+          .validate_fn = validate_subvols_per_directory,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.readdir-optimize",
+          .voltype    = "cluster/distribute",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.nufa",
+          .voltype    = "cluster/distribute",
+          .option     = "!nufa",
+          .type       = NO_DOC,
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
         },
 
 
         /* AFR xlator options */
-        { .key           = "cluster.entry-change-log",
-          .voltype       = "cluster/replicate",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.read-subvolume",
-          .voltype       = "cluster/replicate",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.read-subvolume-index",
-          .voltype       = "cluster/replicate",
-          .op_version    = 2,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.read-hash-mode",
-          .voltype       = "cluster/replicate",
-          .op_version    = 2,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.background-self-heal-count",
-          .voltype       = "cluster/replicate",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.metadata-self-heal",
-          .voltype       = "cluster/replicate",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.data-self-heal",
-          .voltype       = "cluster/replicate",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.entry-self-heal",
-          .voltype       = "cluster/replicate",
-          .op_version    = 1,
-          .client_option = _gf_true
+        { .key        = "cluster.entry-change-log",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.read-subvolume",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags     = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.read-subvolume-index",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.read-hash-mode",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.background-self-heal-count",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.metadata-self-heal",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.data-self-heal",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.entry-self-heal",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
         },
         { .key           = "cluster.self-heal-daemon",
           .voltype       = "cluster/replicate",
           .option        = "!self-heal-daemon",
           .op_version    = 1
         },
-        { .key           = "cluster.heal-timeout",
-          .voltype       = "cluster/replicate",
-          .option        = "!heal-timeout",
-          .op_version    = 2,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.strict-readdir",
-          .voltype       = "cluster/replicate",
-          .type          = NO_DOC,
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.self-heal-window-size",
-          .voltype       = "cluster/replicate",
-          .option        = "data-self-heal-window-size",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.data-change-log",
-          .voltype       = "cluster/replicate",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.metadata-change-log",
-          .voltype       = "cluster/replicate",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.data-self-heal-algorithm",
-          .voltype       = "cluster/replicate",
-          .option        = "data-self-heal-algorithm",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.eager-lock",
-          .voltype       = "cluster/replicate",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.quorum-type",
-          .voltype       = "cluster/replicate",
-          .option        = "quorum-type",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.quorum-count",
-          .voltype       = "cluster/replicate",
-          .option        = "quorum-count",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.choose-local",
-          .voltype       = "cluster/replicate",
-          .op_version    = 2,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.self-heal-readdir-size",
-          .voltype       = "cluster/replicate",
-          .op_version    = 2,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.post-op-delay-secs",
-          .voltype       = "cluster/replicate",
-          .type          = NO_DOC,
-          .op_version    = 2,
-          .client_option = _gf_true
-        },
-        { .key           = "cluster.readdir-failover",
-          .voltype       = "cluster/replicate",
-          .op_version    = 2,
-          .client_option = _gf_true
+        { .key        = "cluster.heal-timeout",
+          .voltype    = "cluster/replicate",
+          .option     = "!heal-timeout",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.strict-readdir",
+          .voltype    = "cluster/replicate",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.self-heal-window-size",
+          .voltype    = "cluster/replicate",
+          .option     = "data-self-heal-window-size",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.data-change-log",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.metadata-change-log",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.data-self-heal-algorithm",
+          .voltype    = "cluster/replicate",
+          .option     = "data-self-heal-algorithm",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.eager-lock",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.quorum-type",
+          .voltype    = "cluster/replicate",
+          .option     = "quorum-type",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.quorum-count",
+          .voltype    = "cluster/replicate",
+          .option     = "quorum-count",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.choose-local",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.self-heal-readdir-size",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.post-op-delay-secs",
+          .voltype    = "cluster/replicate",
+          .type       = NO_DOC,
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.readdir-failover",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
         },
 
         /* Stripe xlator options */
-        { .key           = "cluster.stripe-block-size",
-          .voltype       = "cluster/stripe",
-          .option        = "block-size",
-          .op_version    = 1,
+        { .key         = "cluster.stripe-block-size",
+          .voltype     = "cluster/stripe",
+          .option      = "block-size",
+          .op_version  = 1,
           .validate_fn = validate_stripe,
-          .client_option = _gf_true
+          .flags       = OPT_FLAG_CLIENT_OPT
         },
-        { .key           = "cluster.stripe-coalesce",
-          .voltype       = "cluster/stripe",
-          .option        = "coalesce",
-          .op_version    = 2,
-          .client_option = _gf_true
+        { .key        = "cluster.stripe-coalesce",
+          .voltype    = "cluster/stripe",
+          .option     = "coalesce",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
         },
 
         /* IO-stats xlator options */
@@ -545,55 +545,55 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .option      = "!brick-log-level",
           .op_version  = 1
         },
-        { .key           = "diagnostics.client-log-level",
-          .voltype       = "debug/io-stats",
-          .option        = "!client-log-level",
-          .op_version    = 1,
-          .client_option = _gf_true
+        { .key        = "diagnostics.client-log-level",
+          .voltype    = "debug/io-stats",
+          .option     = "!client-log-level",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
         },
         { .key         = "diagnostics.brick-sys-log-level",
           .voltype     = "debug/io-stats",
           .option      = "!sys-log-level",
           .op_version  = 1
         },
-        { .key           = "diagnostics.client-sys-log-level",
-          .voltype       = "debug/io-stats",
-          .option        = "!sys-log-level",
-          .op_version    = 1,
-          .client_option = _gf_true
+        { .key        = "diagnostics.client-sys-log-level",
+          .voltype    = "debug/io-stats",
+          .option     = "!sys-log-level",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
         },
 
         /* IO-cache xlator options */
-        { .key           = "performance.cache-max-file-size",
-          .voltype       = "performance/io-cache",
-          .option        = "max-file-size",
-          .op_version    = 1,
-          .validate_fn   = validate_cache_max_min_size,
-          .client_option = _gf_true
-        },
-        { .key           = "performance.cache-min-file-size",
-          .voltype       = "performance/io-cache",
-          .option        = "min-file-size",
-          .op_version    = 1,
-          .validate_fn   = validate_cache_max_min_size,
-          .client_option = _gf_true
-        },
-        { .key           = "performance.cache-refresh-timeout",
-          .voltype       = "performance/io-cache",
-          .option        = "cache-timeout",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "performance.cache-priority",
-          .voltype       = "performance/io-cache",
-          .option        = "priority",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "performance.cache-size",
-          .voltype       = "performance/io-cache",
-          .op_version    = 1,
-          .client_option = _gf_true
+        { .key         = "performance.cache-max-file-size",
+          .voltype     = "performance/io-cache",
+          .option      = "max-file-size",
+          .op_version  = 1,
+          .validate_fn = validate_cache_max_min_size,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "performance.cache-min-file-size",
+          .voltype     = "performance/io-cache",
+          .option      = "min-file-size",
+          .op_version  = 1,
+          .validate_fn = validate_cache_max_min_size,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.cache-refresh-timeout",
+          .voltype    = "performance/io-cache",
+          .option     = "cache-timeout",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.cache-priority",
+          .voltype    = "performance/io-cache",
+          .option     = "priority",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.cache-size",
+          .voltype    = "performance/io-cache",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
         },
 
         /* IO-threads xlator options */
@@ -628,88 +628,88 @@ struct volopt_map_entry glusterd_volopt_map[] = {
         },
 
         /* Other perf xlators' options */
-        { .key           = "performance.cache-size",
-          .voltype       = "performance/quick-read",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "performance.flush-behind",
-          .voltype       = "performance/write-behind",
-          .option        = "flush-behind",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "performance.write-behind-window-size",
-          .voltype       = "performance/write-behind",
-          .option        = "cache-size",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "performance.strict-o-direct",
-          .voltype       = "performance/write-behind",
-          .option        = "strict-O_DIRECT",
-          .op_version    = 2,
-          .client_option = _gf_true
-        },
-        { .key           = "performance.strict-write-ordering",
-          .voltype       = "performance/write-behind",
-          .option        = "strict-write-ordering",
-          .op_version    = 2,
-          .client_option = _gf_true
-        },
-        { .key           = "performance.read-ahead-page-count",
-          .voltype       = "performance/read-ahead",
-          .option        = "page-count",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "performance.md-cache-timeout",
-          .voltype       = "performance/md-cache",
-          .option        = "md-cache-timeout",
-          .op_version    = 2,
-          .client_option = _gf_true
+        { .key        = "performance.cache-size",
+          .voltype    = "performance/quick-read",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.flush-behind",
+          .voltype    = "performance/write-behind",
+          .option     = "flush-behind",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.write-behind-window-size",
+          .voltype    = "performance/write-behind",
+          .option     = "cache-size",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.strict-o-direct",
+          .voltype    = "performance/write-behind",
+          .option     = "strict-O_DIRECT",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.strict-write-ordering",
+          .voltype    = "performance/write-behind",
+          .option     = "strict-write-ordering",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.read-ahead-page-count",
+          .voltype    = "performance/read-ahead",
+          .option     = "page-count",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.md-cache-timeout",
+          .voltype    = "performance/md-cache",
+          .option     = "md-cache-timeout",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
         },
 
         /* Client xlator options */
-        { .key           = "network.frame-timeout",
-          .voltype       = "protocol/client",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "network.ping-timeout",
-          .voltype       = "protocol/client",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "network.tcp-window-size",
-          .voltype       = "protocol/client",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "features.lock-heal",
-          .voltype       = "protocol/client",
-          .option        = "lk-heal",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "features.grace-timeout",
-          .voltype       = "protocol/client",
-          .option        = "grace-timeout",
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "client.ssl",
-          .voltype       = "protocol/client",
-          .option        = "transport.socket.ssl-enabled",
-          .type          = NO_DOC,
-          .op_version    = 2,
-          .client_option = _gf_true
-        },
-        { .key           = "network.remote-dio",
-          .voltype       = "protocol/client",
-          .option        = "filter-O_DIRECT",
-          .op_version    = 1,
-          .client_option = _gf_true
+        { .key        = "network.frame-timeout",
+          .voltype    = "protocol/client",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "network.ping-timeout",
+          .voltype    = "protocol/client",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "network.tcp-window-size",
+          .voltype    = "protocol/client",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "features.lock-heal",
+          .voltype    = "protocol/client",
+          .option     = "lk-heal",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "features.grace-timeout",
+          .voltype    = "protocol/client",
+          .option     = "grace-timeout",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "client.ssl",
+          .voltype    = "protocol/client",
+          .option     = "transport.socket.ssl-enabled",
+          .type       = NO_DOC,
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "network.remote-dio",
+          .voltype    = "protocol/client",
+          .option     = "filter-O_DIRECT",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
         },
 
         /* Server xlator options */
@@ -774,132 +774,137 @@ struct volopt_map_entry glusterd_volopt_map[] = {
         },
 
         /* Performance xlators enable/disbable options */
-        { .key           = "performance.write-behind",
-          .voltype       = "performance/write-behind",
-          .option        = "!perf",
-          .value         = "on",
-          .op_version    = 1,
-          .description   = "enable/disable write-behind translator in the "
-                           "volume.",
-          .client_option = _gf_true
-        },
-        { .key           = "performance.read-ahead",
-          .voltype       = "performance/read-ahead",
-          .option        = "!perf",
-          .value         = "on",
-          .op_version    = 1,
-          .description   = "enable/disable read-ahead translator in the "
-                           "volume.",
-          .client_option = _gf_true
-        },
-        { .key           = "performance.io-cache",
-          .voltype       = "performance/io-cache",
-          .option        = "!perf",
-          .value         = "on",
-          .op_version    = 1,
-          .description   = "enable/disable io-cache translator in the volume.",
-          .client_option = _gf_true
-        },
-        { .key           = "performance.quick-read",
-          .voltype       = "performance/quick-read",
-          .option        = "!perf",
-          .value         = "on",
-          .op_version    = 1,
-          .description   = "enable/disable quick-read translator in the "
-                           "volume.",
-          .client_option = _gf_true
-        },
-        { .key           = "performance.open-behind",
-          .voltype       = "performance/open-behind",
-          .option        = "!perf",
-          .value         = "on",
-          .op_version    = 2,
-          .description   = "enable/disable open-behind translator in the "
-                           "volume.",
-          .client_option = _gf_true
-        },
-        { .key           = "performance.stat-prefetch",
-          .voltype       = "performance/md-cache",
-          .option        = "!perf",
-          .value         = "on",
-          .op_version    = 1,
-          .description   = "enable/disable meta-data caching translator in the "
-                           "volume.",
-          .client_option = _gf_true
-        },
-        { .key           = "performance.client-io-threads",
-          .voltype       = "performance/io-threads",
-          .option        = "!perf",
-          .value         = "off",
-          .op_version    = 1,
-          .description   = "enable/disable io-threads translator in the client "
-                           "graph of volume.",
-          .client_option = _gf_true
-        },
-        { .key         = "performance.nfs.write-behind",
+        { .key         = "performance.write-behind",
           .voltype     = "performance/write-behind",
-          .option      = "!nfsperf",
+          .option      = "!perf",
           .value       = "on",
-          .type        = NO_DOC,
-          .op_version  = 1
+          .op_version  = 1,
+          .description = "enable/disable write-behind translator in the "
+                         "volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
         },
-        { .key         = "performance.nfs.read-ahead",
+        { .key         = "performance.read-ahead",
           .voltype     = "performance/read-ahead",
-          .option      = "!nfsperf",
-          .value       = "off",
-          .type        = NO_DOC,
-          .op_version  = 1
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable read-ahead translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
         },
-        { .key         = "performance.nfs.io-cache",
+        { .key         = "performance.io-cache",
           .voltype     = "performance/io-cache",
-          .option      = "!nfsperf",
-          .value       = "off",
-          .type        = NO_DOC,
-          .op_version  = 1
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable io-cache translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT
         },
-        { .key         = "performance.nfs.quick-read",
+        { .key         = "performance.quick-read",
           .voltype     = "performance/quick-read",
-          .option      = "!nfsperf",
-          .value       = "off",
-          .type        = NO_DOC,
-          .op_version  = 1
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable quick-read translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+
+        },
+        { .key         = "performance.open-behind",
+          .voltype     = "performance/open-behind",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 2,
+          .description = "enable/disable open-behind translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+
         },
-        { .key         = "performance.nfs.stat-prefetch",
+        { .key         = "performance.stat-prefetch",
           .voltype     = "performance/md-cache",
-          .option      = "!nfsperf",
-          .value       = "off",
-          .type        = NO_DOC,
-          .op_version  = 1
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable meta-data caching translator in the "
+                         "volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
         },
-        { .key         = "performance.nfs.io-threads",
+        { .key         = "performance.client-io-threads",
           .voltype     = "performance/io-threads",
-          .option      = "!nfsperf",
+          .option      = "!perf",
           .value       = "off",
-          .type        = NO_DOC,
-          .op_version  = 1
-        },
-        { .key           = "performance.force-readdirp",
-          .voltype       = "performance/md-cache",
-          .option        = "force-readdirp",
-          .op_version    = 2,
-          .client_option = _gf_true
+          .op_version  = 1,
+          .description = "enable/disable io-threads translator in the client "
+                         "graph of volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.write-behind",
+          .voltype    = "performance/write-behind",
+          .option     = "!nfsperf",
+          .value      = "on",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.read-ahead",
+          .voltype    = "performance/read-ahead",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.io-cache",
+          .voltype    = "performance/io-cache",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.quick-read",
+          .voltype    = "performance/quick-read",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.stat-prefetch",
+          .voltype    = "performance/md-cache",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.io-threads",
+          .voltype    = "performance/io-threads",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.force-readdirp",
+          .voltype    = "performance/md-cache",
+          .option     = "force-readdirp",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
         },
 
         /* Quota xlator options */
-        { .key           = VKEY_FEATURES_LIMIT_USAGE,
-          .voltype       = "features/quota",
-          .option        = "limit-set",
-          .type          = NO_DOC,
-          .op_version    = 1,
-          .client_option = _gf_true
-        },
-        { .key           = "features.quota-timeout",
-          .voltype       = "features/quota",
-          .option        = "timeout",
-          .value         = "0",
-          .op_version    = 1,
-          .validate_fn   = validate_quota,
-          .client_option = _gf_true
+        { .key        = VKEY_FEATURES_LIMIT_USAGE,
+          .voltype    = "features/quota",
+          .option     = "limit-set",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "features.quota-timeout",
+          .voltype     = "features/quota",
+          .option      = "timeout",
+          .value       = "0",
+          .op_version  = 1,
+          .validate_fn = validate_quota,
+          .flags       = OPT_FLAG_CLIENT_OPT
         },
 
         /* Marker xlator options */
@@ -929,12 +934,13 @@ struct volopt_map_entry glusterd_volopt_map[] = {
         },
 
         /* Debug xlators options */
-        { .key         = "debug.trace",
-          .voltype     = "debug/trace",
-          .option      = "!debug",
-          .value       = "off",
-          .type        = NO_DOC,
-          .op_version  = 1
+        { .key        = "debug.trace",
+          .voltype    = "debug/trace",
+          .option     = "!debug",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
         },
         { .key         = "debug.log-history",
           .voltype     = "debug/trace",
@@ -960,12 +966,13 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .type        = NO_DOC,
           .op_version  = 2
         },
-        { .key         = "debug.error-gen",
-          .voltype     = "debug/error-gen",
-          .option      = "!debug",
-          .value       = "off",
-          .type        = NO_DOC,
-          .op_version  = 1
+        { .key        = "debug.error-gen",
+          .voltype    = "debug/error-gen",
+          .option     = "!debug",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
         },
 
 
@@ -1096,19 +1103,19 @@ struct volopt_map_entry glusterd_volopt_map[] = {
         },
 
         /* Other options which don't fit any place above */
-        { .key           = "features.read-only",
-          .voltype       = "features/read-only",
-          .option        = "!read-only",
-          .value         = "off",
-          .op_version    = 2,
-          .client_option = _gf_true
-        },
-        { .key           = "features.worm",
-          .voltype       = "features/worm",
-          .option        = "!worm",
-          .value         = "off",
-          .op_version    = 2,
-          .client_option = _gf_true
+        { .key        = "features.read-only",
+          .voltype    = "features/read-only",
+          .option     = "!read-only",
+          .value      = "off",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "features.worm",
+          .voltype    = "features/worm",
+          .option     = "!worm",
+          .value      = "off",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
         },
         { .key         = "storage.linux-aio",
           .voltype     = "storage/posix",
@@ -1124,11 +1131,11 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .option      = "brick-gid",
           .op_version  = 2
         },
-        { .key           = "config.memory-accounting",
-          .voltype       = "configuration",
-          .option        = "!config",
-          .op_version    = 2,
-          .client_option = _gf_true
+        { .key        = "config.memory-accounting",
+          .voltype    = "configuration",
+          .option     = "!config",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
         },
         { .key         = "config.transport",
           .voltype     = "configuration",