protocol/client: add an option to filter O_DIRECT flag in open

with the option, the idea is all client-side caching will be disabled,
where as on server side process, the fd will be treated as a regular
fd, thus helping the performance better.

"gluster volume set <VOLNAME> remote-dio enable" would set
this option in client protocol volumes.

Change-Id: Id2255a167137f8fee20849513e3011274dc829b4
Signed-off-by: Amar Tumballi <amarts@redhat.com>
BUG: 845213
Reviewed-on: http://review.gluster.org/4206
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

4 files changed, 46 insertions, 2 deletions

diff --git a/tests/bugs/bug-845213.t b/tests/bugs/bug-845213.t
new file mode 100644
index 000000000..e79b37109
--- /dev/null
+++ b/tests/bugs/bug-845213.t
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+
+cleanup;
+
+
+## Start and create a volume
+TEST glusterd;
+TEST pidof glusterd;
+TEST $CLI volume info;
+
+## Create and start a volume with aio enabled
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2};
+TEST $CLI volume set $V0 remote-dio enable;
+TEST $CLI volume set $V0 network.remote-dio disable;
+
+cleanup;
+
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 3629361ad..76172dd9b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -178,6 +178,7 @@ static struct volopt_map_entry glusterd_volopt_map[] = {
         {"features.lock-heal",                   "protocol/client",           "lk-heal", NULL, DOC, 0, 1},
         {"features.grace-timeout",               "protocol/client",           "grace-timeout", NULL, DOC, 0, 1},
         {"client.ssl",                           "protocol/client",           "transport.socket.ssl-enabled", NULL, NO_DOC, 0, 2},
+        {"network.remote-dio",                   "protocol/client",           "filter-O_DIRECT", NULL, DOC, 0, 1},
 
         /* Server xlator options */
         {"network.tcp-window-size",              "protocol/server",           NULL, NULL, NO_DOC, 0, 1},
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 6cc3c9896..46fe622db 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -815,12 +815,16 @@ client_create (call_frame_t *frame, xlator_t *this, loc_t *loc,  int32_t flags,
                 goto out;
 
         args.loc = loc;
-        args.flags = flags;
         args.mode = mode;
         args.fd = fd;
         args.umask = umask;
         args.xdata = xdata;
 
+        if (!conf->filter_o_direct)
+                args.flags = flags;
+        else
+                args.flags = (flags & ~O_DIRECT);
+
         proc = &conf->fops->proctable[GF_FOP_CREATE];
         if (!proc) {
                 gf_log (this->name, GF_LOG_ERROR,
@@ -854,10 +858,14 @@ client_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto out;
 
         args.loc = loc;
-        args.flags = flags;
         args.fd = fd;
         args.xdata = xdata;
 
+        if (!conf->filter_o_direct)
+                args.flags = flags;
+        else
+                args.flags = (flags & ~O_DIRECT);
+
         proc = &conf->fops->proctable[GF_FOP_OPEN];
         if (!proc) {
                 gf_log (this->name, GF_LOG_ERROR,
@@ -2198,6 +2206,9 @@ build_client_config (xlator_t *this, clnt_conf_t *conf)
                 gf_log (this->name, GF_LOG_WARNING,
                         "option 'remote-subvolume' not given");
 
+        GF_OPTION_INIT ("filter-O_DIRECT", conf->filter_o_direct,
+                        bool, out);
+
         ret = 0;
 out:
         return ret;
@@ -2382,6 +2393,9 @@ reconfigure (xlator_t *this, dict_t *options)
                 }
         }
 
+        GF_OPTION_RECONF ("filter-O_DIRECT", conf->filter_o_direct,
+                          options, bool, out);
+
         ret = client_init_grace_timer (this, options, conf);
         if (ret)
                 goto out;
@@ -2717,5 +2731,13 @@ struct volume_options options[] = {
          .min  = GF_MIN_SOCKET_WINDOW_SIZE,
          .max  = GF_MAX_SOCKET_WINDOW_SIZE
         },
+        { .key   = {"filter-O_DIRECT"},
+          .type  = GF_OPTION_TYPE_BOOL,
+          .default_value = "disable",
+          .description = "If enabled, in open() and creat() calls, O_DIRECT "
+          "flag will be filtered at the client protocol level so server will "
+          "still continue to cache the file. This works similar to NFS's "
+          "behavior of O_DIRECT",
+        },
         { .key   = {NULL} },
 };
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index 420674910..9241eefa3 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -114,6 +114,8 @@ typedef struct clnt_conf {
 						   the reconnection happen after
 						   the usual 3-second wait
 						*/
+        gf_boolean_t           filter_o_direct; /* if set, filter O_DIRECT from
+                                                   the flags list of open() */
 } clnt_conf_t;
 
 typedef struct _client_fd_ctx {