summaryrefslogtreecommitdiffstats
path: root/xlators/protocol/client/src
diff options
context:
space:
mode:
authorMohammed Junaid <junaid@redhat.com>2012-02-08 18:06:39 +0530
committerVijay Bellur <vijay@gluster.com>2012-02-20 04:45:31 -0800
commitf764516c2e526624ce0088963924ff2d88304553 (patch)
tree85262797baad440b12853a3a6ad41ab518d9f996 /xlators/protocol/client/src
parent4d1b040f00e7ec8de997d151b35fa035bba9cb25 (diff)
protocol/client,server: fcntl lock self healing.
Currently(with out this patch), on a disconnect the server cleans up the transport which inturn closes the fd's and releases the locks acquired on those fd's by that client. On a reconnect, client just reopens the fd's but doesn't reacquire the locks. The application that had previously acquired the locks still is under the assumption that it is the owner of those locks which might have been granted to other clients(if they request) by the server leading to data corruption. This patch allows the client to reacquire the fcntl locks (held on the fd's) during client-server handshake. * The server identifies the client via process-uuid-xl (which is a combination of uuid and client-protocol name, it is assumed to be unique) and lk-version number. * The client maintains a list of process-uuid-xl, lk-version pair for each accepted connection. On a connect, the server traverses the list for a matching pair, if a matching pair is not found the the server returns lk-version with value 0, else it returns the lk-version it has in store. * On a disconnect, the server and client enter grace period, and on the completion of the grace period, the client bumps up its lk-version number (which means, it will reacquire the locks the next time) and the server will distroy the connection. If reconnection happens within the grace period, the server will find the matching (process-uuid-xl, lk-version) pair in its list which guarantees that the fd's and there corresponding locks are still valid for this client. Configurable options: To set grace-timeout, the following options are option server.grace-timeout value option client.grace-timeout value To enable or disable the lk-heal, option lk-heal [on|off] gluster volume set command can be used to configurable options Change-Id: Id677ef1087b300d649f278b8b2aa0d94eae85ed2 BUG: 795386 Signed-off-by: Mohammed Junaid <junaid@redhat.com> Reviewed-on: http://review.gluster.com/2766 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Vijay Bellur <vijay@gluster.com>
Diffstat (limited to 'xlators/protocol/client/src')
-rw-r--r--xlators/protocol/client/src/client-handshake.c481
-rw-r--r--xlators/protocol/client/src/client-lk.c1
-rw-r--r--xlators/protocol/client/src/client-mem-types.h1
-rw-r--r--xlators/protocol/client/src/client.c165
-rw-r--r--xlators/protocol/client/src/client.h16
-rw-r--r--xlators/protocol/client/src/client3_1-fops.c16
6 files changed, 654 insertions, 26 deletions
diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index 91cda6d0c..1896e6b63 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -22,6 +22,7 @@
#include "config.h"
#endif
+#include "fd-lk.h"
#include "client.h"
#include "xlator.h"
#include "defaults.h"
@@ -39,6 +40,18 @@ extern rpc_clnt_prog_t clnt_pmap_prog;
int client_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
void *myframe);
+int client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe);
+
+int client_set_lk_version (xlator_t *this);
+
+typedef struct client_fd_lk_local {
+ int ref;
+ gf_boolean_t error;
+ gf_lock_t lock;
+ clnt_fd_ctx_t *fdctx;
+}clnt_fd_lk_local_t;
+
/* Handshake */
void
@@ -391,6 +404,411 @@ client_notify_parents_child_up (xlator_t *this)
}
int
+client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ int32_t ret = -1;
+ call_frame_t *fr = NULL;
+ gf_set_lk_ver_rsp rsp = {0,};
+
+ fr = (call_frame_t *) myframe;
+ GF_VALIDATE_OR_GOTO ("client", fr, out);
+
+ if (req->rpc_status == -1) {
+ gf_log (fr->this->name, GF_LOG_WARNING,
+ "received RPC status error");
+ goto out;
+ }
+
+ ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_set_lk_ver_rsp);
+ if (ret < 0)
+ gf_log (fr->this->name, GF_LOG_WARNING,
+ "xdr decoding failed");
+ else
+ gf_log (fr->this->name, GF_LOG_DEBUG,
+ "Server lk version = %d", rsp.lk_ver);
+
+ ret = 0;
+out:
+ if (fr)
+ STACK_DESTROY (fr->root);
+
+ return ret;
+}
+
+int
+client_set_lk_version (xlator_t *this)
+{
+ int ret = -1;
+ clnt_conf_t *conf = NULL;
+ call_frame_t *frame = NULL;
+ gf_set_lk_ver_req req = {0, };
+
+ conf = (clnt_conf_t *) this->private;
+
+ req.lk_ver = client_get_lk_ver (conf);
+ req.uid = this->ctx->process_uuid;
+
+ gf_log (this->name, GF_LOG_DEBUG, "Sending SET_LK_VERSION");
+
+ frame = create_frame (this, this->ctx->pool);
+ if (!frame)
+ goto out;
+
+ ret = client_submit_request (this, &req, frame,
+ conf->handshake,
+ GF_HNDSK_SET_LK_VER,
+ client_set_lk_version_cbk,
+ NULL, NULL, 0, NULL, 0, NULL,
+ (xdrproc_t)xdr_gf_set_lk_ver_req);
+out:
+ if (ret < 0)
+ gf_log (this->name, GF_LOG_WARNING,
+ "Failed to send SET_LK_VERSION to server");
+
+ return ret;
+}
+
+int
+client_fd_lk_list_empty (fd_lk_ctx_t *lk_ctx)
+{
+ int ret = 1;
+
+ GF_VALIDATE_OR_GOTO ("client", lk_ctx, out);
+
+ LOCK (&lk_ctx->lock);
+ {
+ ret = list_empty (&lk_ctx->lk_list);
+ }
+ UNLOCK (&lk_ctx->lock);
+out:
+ return ret;
+}
+
+int
+client_fd_lk_count (fd_lk_ctx_t *lk_ctx)
+{
+ int count = 0;
+ fd_lk_ctx_node_t *fd_lk = NULL;
+
+ GF_VALIDATE_OR_GOTO ("client", lk_ctx, err);
+
+ LOCK (&lk_ctx->lock);
+ {
+ list_for_each_entry (fd_lk, &lk_ctx->lk_list, next)
+ count++;
+ }
+ UNLOCK (&lk_ctx->lock);
+
+ return count;
+err:
+ return -1;
+}
+
+clnt_fd_lk_local_t *
+clnt_fd_lk_local_ref (xlator_t *this, clnt_fd_lk_local_t *local)
+{
+ GF_VALIDATE_OR_GOTO (this->name, local, out);
+
+ LOCK (&local->lock);
+ {
+ local->ref++;
+ }
+ UNLOCK (&local->lock);
+out:
+ return local;
+}
+
+int
+clnt_fd_lk_local_unref (xlator_t *this, clnt_fd_lk_local_t *local)
+{
+ int ref = -1;
+
+ GF_VALIDATE_OR_GOTO (this->name, local, out);
+
+ LOCK (&local->lock);
+ {
+ ref = --local->ref;
+ }
+ UNLOCK (&local->lock);
+
+ if (ref == 0) {
+ LOCK_DESTROY (&local->lock);
+ GF_FREE (local);
+ }
+ ref = 0;
+out:
+ return ref;
+}
+
+clnt_fd_lk_local_t *
+clnt_fd_lk_local_create (clnt_fd_ctx_t *fdctx)
+{
+ clnt_fd_lk_local_t *local = NULL;
+
+ local = GF_CALLOC (1, sizeof (clnt_fd_lk_local_t),
+ gf_client_mt_clnt_fd_lk_local_t);
+ if (!local)
+ goto out;
+
+ local->ref = 1;
+ local->error = _gf_false;
+ local->fdctx = fdctx;
+
+ LOCK_INIT (&local->lock);
+out:
+ return local;
+}
+
+void
+clnt_mark_fd_bad (clnt_conf_t *conf, clnt_fd_ctx_t *fdctx)
+{
+ pthread_mutex_lock (&conf->lock);
+ {
+ fdctx->remote_fd = -1;
+ }
+ pthread_mutex_unlock (&conf->lock);
+}
+
+// call decrement_reopen_fd_count
+int
+clnt_release_reopen_fd_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ xlator_t *this = NULL;
+ call_frame_t *frame = NULL;
+ clnt_conf_t *conf = NULL;
+ clnt_fd_ctx_t *fdctx = NULL;
+
+ frame = myframe;
+ this = frame->this;
+ fdctx = (clnt_fd_ctx_t *) frame->local;
+ conf = (clnt_conf_t *) this->private;
+
+ clnt_mark_fd_bad (conf, fdctx);
+
+ decrement_reopen_fd_count (this, conf);
+
+ frame->local = NULL;
+ STACK_DESTROY (frame->root);
+
+ return 0;
+}
+
+int
+clnt_release_reopen_fd (xlator_t *this, clnt_fd_ctx_t *fdctx)
+{
+ int ret = -1;
+ clnt_conf_t *conf = NULL;
+ call_frame_t *frame = NULL;
+ gfs3_release_req req = {{0,},};
+
+ conf = (clnt_conf_t *) this->private;
+
+ frame = create_frame (THIS, THIS->ctx->pool);
+ if (!frame)
+ goto out;
+
+ frame->local = (void *) fdctx;
+ req.fd = fdctx->remote_fd;
+
+ ret = client_submit_request (this, &req, frame, conf->fops,
+ GFS3_OP_RELEASE,
+ clnt_release_reopen_fd_cbk, NULL,
+ NULL, 0, NULL, 0, NULL,
+ (xdrproc_t)xdr_gfs3_releasedir_req);
+out:
+ if (ret) {
+ decrement_reopen_fd_count (this, conf);
+ clnt_mark_fd_bad (conf, fdctx);
+ if (frame) {
+ frame->local = NULL;
+ STACK_DESTROY (frame->root);
+ }
+ }
+
+ return 0;
+}
+
+int
+clnt_fd_lk_local_mark_error (xlator_t *this,
+ clnt_fd_lk_local_t *local)
+{
+ gf_boolean_t error = _gf_false;
+
+ LOCK (&local->lock);
+ {
+ error = local->error;
+ local->error = _gf_true;
+ }
+ UNLOCK (&local->lock);
+
+ if (error)
+ clnt_release_reopen_fd (this, local->fdctx);
+
+ return 0;
+}
+
+// Also, I think in reopen_cbk, the fdctx is added to
+// saved_fd list.. avoid that, may cause a problem
+// Reason: While the locks on the fd are reacquired, a release
+// fop may be received by the client-protocol translator
+// which will free the fdctx datastructure.
+int
+client_reacquire_lock_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ int32_t ret = -1;
+ xlator_t *this = NULL;
+ gf_common_rsp rsp = {0,};
+ call_frame_t *frame = NULL;
+ clnt_fd_lk_local_t *local = NULL;
+
+ frame = (call_frame_t *) myframe;
+ this = frame->this;
+ local = (clnt_fd_lk_local_t *) frame->local;
+
+ if (req->rpc_status == -1) {
+ gf_log ("client", GF_LOG_WARNING,
+ "request failed at rpc");
+ goto out;
+ }
+
+ ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+ if (ret < 0) {
+ gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
+ goto out;
+ }
+
+ if (rsp.op_ret == -1) {
+ gf_log (this->name, GF_LOG_ERROR, "lock request failed");
+ ret = -1;
+ goto out;
+ }
+
+ // TODO: Add more info to log.
+ gf_log (this->name, GF_LOG_DEBUG, "Reacquired lock");
+
+ ret = 0;
+out:
+ if (ret < 0)
+ clnt_fd_lk_local_mark_error (this, local);
+
+ (void) clnt_fd_lk_local_unref (this, local);
+ frame->local = NULL;
+ STACK_DESTROY (frame->root);
+
+ return ret;
+}
+
+int
+_client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx)
+{
+ int32_t ret = -1;
+ int32_t gf_cmd = 0;
+ int32_t gf_type = 0;
+ gfs3_lk_req req = {{0,},};
+ struct gf_flock flock = {0,};
+ fd_lk_ctx_t *lk_ctx = NULL;
+ clnt_fd_lk_local_t *local = NULL;
+ fd_lk_ctx_node_t *fd_lk = NULL;
+ call_frame_t *frame = NULL;
+ clnt_conf_t *conf = NULL;
+
+ conf = (clnt_conf_t *) this->private;
+ lk_ctx = fdctx->lk_ctx;
+
+ local = clnt_fd_lk_local_create (fdctx);
+ if (!local) {
+ clnt_release_reopen_fd (this, fdctx);
+ goto out;
+ }
+
+ list_for_each_entry (fd_lk, &lk_ctx->lk_list, next) {
+ memcpy (&flock, &fd_lk->user_flock,
+ sizeof (struct gf_flock));
+
+ ret = client_cmd_to_gf_cmd (fd_lk->cmd, &gf_cmd);
+ if (ret) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "client_cmd_to_gf_cmd failed, "
+ "aborting reacquiring of locks");
+ break;
+ }
+
+ gf_type = client_type_to_gf_type (flock.l_type);
+ req.fd = fdctx->remote_fd;
+ req.cmd = gf_cmd;
+ req.type = gf_type;
+ (void) gf_proto_flock_from_flock (&req.flock,
+ &flock);
+
+ memcpy (req.gfid, fdctx->inode->gfid, 16);
+
+ frame = create_frame (THIS, THIS->ctx->pool);
+ if (!frame) {
+ ret = -1;
+ break;
+ }
+
+ frame->local = clnt_fd_lk_local_ref (this, local);
+ frame->root->lk_owner = fd_lk->user_flock.l_owner;
+
+ ret = client_submit_request (this, &req, frame,
+ conf->fops, GFS3_OP_LK,
+ client_reacquire_lock_cbk,
+ NULL, NULL, 0, NULL, 0, NULL,
+ (xdrproc_t)xdr_gfs3_lk_req);
+ if (ret)
+ break;
+
+ ret = 0;
+ frame = NULL;
+ }
+
+ if (ret) {
+ clnt_fd_lk_local_mark_error (this, local);
+
+ if (frame) {
+ if (frame->local) {
+ clnt_fd_lk_local_unref (this, frame->local);
+ frame->local = NULL;
+ }
+ STACK_DESTROY (frame->root);
+ }
+ }
+ if (local)
+ (void) clnt_fd_lk_local_unref (this, local);
+out:
+ return ret;
+}
+
+int
+client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx)
+{
+ int32_t ret = -1;
+ fd_lk_ctx_t *lk_ctx = NULL;
+
+ if (client_fd_lk_list_empty (fdctx->lk_ctx)) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "fd lock list is empty");
+ decrement_reopen_fd_count (this, (clnt_conf_t *)this->private);
+ ret = 0;
+ goto out;
+ }
+
+ lk_ctx = fdctx->lk_ctx;
+
+ LOCK (&lk_ctx->lock);
+ {
+ ret = _client_reacquire_lock (this, fdctx);
+ }
+ UNLOCK (&lk_ctx->lock);
+out:
+ return ret;
+}
+
+int
client3_1_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
void *myframe)
{
@@ -402,11 +820,13 @@ client3_1_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
clnt_conf_t *conf = NULL;
clnt_fd_ctx_t *fdctx = NULL;
call_frame_t *frame = NULL;
+ xlator_t *this = NULL;
frame = myframe;
if (!frame || !frame->this)
goto out;
+ this = frame->this;
local = frame->local;
conf = frame->this->private;
@@ -454,7 +874,7 @@ client3_1_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
fdctx->remote_fd = rsp.fd;
if (!fdctx->released) {
list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
- if (!list_empty (&fdctx->lock_list))
+ if (!client_fd_lk_list_empty (fdctx->lk_ctx))
attempt_lock_recovery = _gf_true;
fdctx = NULL;
}
@@ -463,31 +883,27 @@ client3_1_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
ret = 0;
- attempt_lock_recovery = _gf_false; /* temporarily */
-
- if (attempt_lock_recovery) {
- ret = client_attempt_lock_recovery (frame->this, local->fdctx);
- if (ret < 0) {
- gf_log (frame->this->name, GF_LOG_DEBUG,
- "lock recovery not attempted on fd");
- } else {
- gf_log (frame->this->name, GF_LOG_INFO,
- "need to attempt lock recovery on %"PRIu64
- " open fds", fd_count);
- }
+ if (conf->lk_heal && attempt_lock_recovery) {
+ /* Delay decrement the reopen fd count untill all the
+ locks corresponding to this fd are acquired.*/
+ gf_log (frame->this->name, GF_LOG_WARNING, "acquiring locks on "
+ "%s", local->loc.path);
+ ret = client_reacquire_lock (frame->this, local->fdctx);
} else {
fd_count = decrement_reopen_fd_count (frame->this, conf);
}
out:
if (fdctx)
- client_fdctx_destroy (frame->this, fdctx);
+ client_fdctx_destroy (this, fdctx);
if ((ret < 0) && frame && frame->this && conf)
decrement_reopen_fd_count (frame->this, conf);
- frame->local = NULL;
- STACK_DESTROY (frame->root);
+ if (frame) {
+ frame->local = NULL;
+ STACK_DESTROY (frame->root);
+ }
client_local_wipe (local);
@@ -792,7 +1208,8 @@ client_post_handshake (call_frame_t *frame, xlator_t *this)
}
} else {
gf_log (this->name, GF_LOG_DEBUG,
- "no open fds - notifying all parents child up");
+ "no fds to open - notifying all parents child up");
+ client_set_lk_version (this);
client_notify_parents_child_up (this);
}
out:
@@ -814,6 +1231,7 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
int32_t op_ret = 0;
int32_t op_errno = 0;
gf_boolean_t auth_fail = _gf_false;
+ uint32_t lk_ver = 0;
frame = myframe;
this = frame->this;
@@ -895,6 +1313,15 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
goto out;
}
+ ret = dict_get_uint32 (reply, "clnt-lk-version", &lk_ver);
+ if (ret) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "failed to find key 'clnt-lk-version' in the options");
+ goto out;
+ }
+
+ gf_log (this->name, GF_LOG_INFO, "clnt-lk-version = %d, "
+ "server-lk-version = %d", client_get_lk_ver (conf), lk_ver);
/* TODO: currently setpeer path is broken */
/*
if (process_uuid && req->conn &&
@@ -930,8 +1357,15 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
conf->need_different_port = 0;
- /* TODO: more to test */
- client_post_handshake (frame, frame->this);
+ if (lk_ver != client_get_lk_ver (conf)) {
+ client_mark_fd_bad (this);
+ client_post_handshake (frame, frame->this);
+ } else {
+ /*TODO: Traverse the saved fd list, and send
+ release to the server on fd's that were closed
+ during grace period */
+ ;
+ }
out:
if (auth_fail) {
@@ -1043,6 +1477,14 @@ client_setvolume (xlator_t *this, struct rpc_clnt *rpc)
"failed to set 'volfile-checksum'");
}
+ ret = dict_set_int16 (options, "clnt-lk-version",
+ client_get_lk_ver (conf));
+ if (ret < 0) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "failed to set clnt-lk-version(%"PRIu32") in handshake msg",
+ client_get_lk_ver (conf));
+ }
+
req.dict.dict_len = dict_serialized_length (options);
if (req.dict.dict_len < 0) {
gf_log (this->name, GF_LOG_ERROR,
@@ -1366,6 +1808,7 @@ char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
[GF_HNDSK_SETVOLUME] = "SETVOLUME",
[GF_HNDSK_GETSPEC] = "GETSPEC",
[GF_HNDSK_PING] = "PING",
+ [GF_HNDSK_SET_LK_VER] = "SET_LK_VER"
};
rpc_clnt_prog_t clnt_handshake_prog = {
diff --git a/xlators/protocol/client/src/client-lk.c b/xlators/protocol/client/src/client-lk.c
index 842e3ec5b..e99fe774d 100644
--- a/xlators/protocol/client/src/client-lk.c
+++ b/xlators/protocol/client/src/client-lk.c
@@ -608,6 +608,7 @@ decrement_reopen_fd_count (xlator_t *this, clnt_conf_t *conf)
if (fd_count == 0) {
gf_log (this->name, GF_LOG_INFO,
"last fd open'd/lock-self-heal'd - notifying CHILD-UP");
+ client_set_lk_version (this);
client_notify_parents_child_up (this);
}
diff --git a/xlators/protocol/client/src/client-mem-types.h b/xlators/protocol/client/src/client-mem-types.h
index c2aa690b1..6bc7daad2 100644
--- a/xlators/protocol/client/src/client-mem-types.h
+++ b/xlators/protocol/client/src/client-mem-types.h
@@ -29,6 +29,7 @@ enum gf_client_mem_types_ {
gf_client_mt_clnt_req_buf_t,
gf_client_mt_clnt_fdctx_t,
gf_client_mt_clnt_lock_t,
+ gf_client_mt_clnt_fd_lk_local_t,
gf_client_mt_end,
};
#endif /* __CLIENT_MEM_TYPES_H__ */
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 229e01917..8955e237d 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -40,6 +40,81 @@ int client_handshake (xlator_t *this, struct rpc_clnt *rpc);
void client_start_ping (void *data);
int client_init_rpc (xlator_t *this);
int client_destroy_rpc (xlator_t *this);
+int client_mark_fd_bad (xlator_t *this);
+
+int32_t
+client_type_to_gf_type (short l_type)
+{
+ int32_t gf_type;
+
+ switch (l_type) {
+ case F_RDLCK:
+ gf_type = GF_LK_F_RDLCK;
+ break;
+ case F_WRLCK:
+ gf_type = GF_LK_F_WRLCK;
+ break;
+ case F_UNLCK:
+ gf_type = GF_LK_F_UNLCK;
+ break;
+ }
+
+ return gf_type;
+}
+
+uint32_t
+client_get_lk_ver (clnt_conf_t *conf)
+{
+ uint32_t lk_ver = 0;
+
+ GF_VALIDATE_OR_GOTO ("client", conf, out);
+
+ pthread_mutex_lock (&conf->lock);
+ {
+ lk_ver = conf->lk_version;
+ }
+ pthread_mutex_unlock (&conf->lock);
+out:
+ return lk_ver;
+}
+
+void
+client_grace_timeout (void *data)
+{
+ int ver = 0;
+ xlator_t *this = NULL;
+ struct clnt_conf *conf = NULL;
+ struct rpc_clnt *rpc = NULL;
+
+ GF_VALIDATE_OR_GOTO ("client", data, out);
+
+ this = THIS;
+
+ rpc = (struct rpc_clnt *) data;
+
+ conf = (struct clnt_conf *) this->private;
+
+ pthread_mutex_lock (&conf->lock);
+ {
+ ver = ++conf->lk_version;
+ /* ver == 0 is a special value used by server
+ to notify client that this is a fresh connect.*/
+ if (ver == 0)
+ ver = ++conf->lk_version;
+
+ gf_timer_call_cancel (this->ctx, conf->grace_timer);
+ conf->grace_timer = NULL;
+ }
+ pthread_mutex_unlock (&conf->lock);
+
+ gf_log (this->name, GF_LOG_WARNING,
+ "client grace timer expired, updating "
+ "the lk-version to %d", ver);
+
+ client_mark_fd_bad (this);
+out:
+ return;
+}
int
client_submit_request (xlator_t *this, void *req, call_frame_t *frame,
@@ -828,7 +903,6 @@ out:
}
-
int32_t
client_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
{
@@ -1455,7 +1529,6 @@ out:
return 0;
}
-
int32_t
client_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
struct gf_flock *lock)
@@ -1841,7 +1914,7 @@ out:
}
- int
+int
client_mark_fd_bad (xlator_t *this)
{
clnt_conf_t *conf = NULL;
@@ -1908,11 +1981,42 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
conf->last_sent_event = GF_EVENT_CHILD_UP;
}
}
+
+ /* Cancel grace timer if set */
+ pthread_mutex_lock (&conf->lock);
+ {
+ if (conf->grace_timer) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "Cancelling the grace timer");
+
+ gf_timer_call_cancel (this->ctx,
+ conf->grace_timer);
+ conf->grace_timer = NULL;
+ }
+ }
+ pthread_mutex_unlock (&conf->lock);
+
break;
}
case RPC_CLNT_DISCONNECT:
+ /* client_mark_fd_bad (this); */
- client_mark_fd_bad (this);
+ pthread_mutex_lock (&conf->lock);
+ {
+ if (conf->grace_timer) {
+ gf_log (this->name, GF_LOG_DEBUG,
+ "Client grace timer is already set");
+ } else {
+ gf_log (this->name, GF_LOG_WARNING,
+ "Registering a grace timer");
+ conf->grace_timer =
+ gf_timer_call_after (this->ctx,
+ conf->grace_tv,
+ client_grace_timeout,
+ conf->rpc);
+ }
+ }
+ pthread_mutex_unlock (&conf->lock);
if (!conf->skip_notify) {
if (conf->connected)
@@ -2107,6 +2211,40 @@ out:
int
+client_init_grace_timer (xlator_t *this, dict_t *options,
+ clnt_conf_t *conf)
+{
+ char *lk_heal = NULL;
+ int32_t ret = -1;
+ int32_t grace_timeout = -1;
+
+ GF_VALIDATE_OR_GOTO ("client", this, out);
+ GF_VALIDATE_OR_GOTO (this->name, options, out);
+ GF_VALIDATE_OR_GOTO (this->name, conf, out);
+
+ conf->lk_heal = _gf_true;
+
+ ret = dict_get_str (options, "lk-heal", &lk_heal);
+ if (!ret)
+ gf_string2boolean (lk_heal, &conf->lk_heal);
+
+ ret = dict_get_int32 (options, "grace-timeout", &grace_timeout);
+ if (!ret)
+ conf->grace_tv.tv_sec = grace_timeout;
+ else
+ conf->grace_tv.tv_sec = 10;
+
+ conf->grace_tv.tv_usec = 0;
+
+ gf_log (this->name, GF_LOG_INFO, "lk-heal = %s",
+ (conf->lk_heal) ? "on" : "off");
+
+ ret = 0;
+out:
+ return ret;
+}
+
+int
reconfigure (xlator_t *this, dict_t *options)
{
clnt_conf_t *conf = NULL;
@@ -2153,6 +2291,10 @@ reconfigure (xlator_t *this, dict_t *options)
}
}
+ ret = client_init_grace_timer (this, options, conf);
+ if (ret)
+ goto out;
+
ret = 0;
out:
return ret;
@@ -2186,6 +2328,14 @@ init (xlator_t *this)
pthread_mutex_init (&conf->lock, NULL);
INIT_LIST_HEAD (&conf->saved_fds);
+ /* Initialize parameters for lock self healing*/
+ conf->lk_version = 1;
+ conf->grace_timer = NULL;
+
+ ret = client_init_grace_timer (this, this->options, conf);
+ if (ret)
+ goto out;
+
LOCK_INIT (&conf->rec_lock);
conf->last_sent_event = -1; /* To start with we don't have any events */
@@ -2207,7 +2357,6 @@ init (xlator_t *this)
goto out;
}
-
ret = client_init_rpc (this);
out:
if (ret)
@@ -2409,5 +2558,11 @@ struct volume_options options[] = {
{ .key = {"client-bind-insecure"},
.type = GF_OPTION_TYPE_BOOL
},
+ { .key = {"lk-heal"},
+ .type = GF_OPTION_TYPE_STR
+ },
+ { .key = {"grace-timeout"},
+ .type = GF_OPTION_TYPE_INT
+ },
{ .key = {NULL} },
};
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index 2dda451c9..00addf34c 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -29,6 +29,7 @@
#include "client-mem-types.h"
#include "protocol-common.h"
#include "glusterfs3.h"
+#include "fd-lk.h"
/* FIXME: Needs to be defined in a common file */
#define CLIENT_CMD_CONNECT "trusted.glusterfs.client-connect"
@@ -91,6 +92,12 @@ typedef struct clnt_conf {
char need_different_port; /* flag used to change the
portmap path in case of
'tcp,rdma' on server */
+ gf_boolean_t lk_heal;
+ uint16_t lk_version; /* this variable is used to distinguish
+ client-server transaction while
+ performing lock healing */
+ struct timeval grace_tv;
+ gf_timer_t *grace_timer;
} clnt_conf_t;
typedef struct _client_fd_ctx {
@@ -105,7 +112,7 @@ typedef struct _client_fd_ctx {
char released;
int32_t flags;
int32_t wbflags;
-
+ fd_lk_ctx_t *lk_ctx;
pthread_mutex_t mutex;
struct list_head lock_list; /* List of all granted locks on this fd */
} clnt_fd_ctx_t;
@@ -211,4 +218,11 @@ int32_t client_dump_locks (char *name, inode_t *inode,
dict_t *dict);
int client_fdctx_destroy (xlator_t *this, clnt_fd_ctx_t *fdctx);
+uint32_t client_get_lk_ver (clnt_conf_t *conf);
+
+int32_t client_type_to_gf_type (short l_type);
+
+int client_mark_fd_bad (xlator_t *this);
+
+int client_set_lk_version (xlator_t *this);
#endif /* !_CLIENT_H */
diff --git a/xlators/protocol/client/src/client3_1-fops.c b/xlators/protocol/client/src/client3_1-fops.c
index 76d4fb0d6..4d6d57528 100644
--- a/xlators/protocol/client/src/client3_1-fops.c
+++ b/xlators/protocol/client/src/client3_1-fops.c
@@ -351,6 +351,7 @@ client3_1_open_cbk (struct rpc_req *req, struct iovec *iov, int count,
fdctx->inode = inode_ref (fd->inode);
fdctx->flags = local->flags;
fdctx->wbflags = local->wbflags;
+ fdctx->lk_ctx = fd_lk_ctx_ref (fd->lk_ctx);
INIT_LIST_HEAD (&fdctx->sfd_pos);
INIT_LIST_HEAD (&fdctx->lock_list);
@@ -2279,17 +2280,30 @@ client3_1_releasedir_cbk (struct rpc_req *req, struct iovec *iov, int count,
int
client_fdctx_destroy (xlator_t *this, clnt_fd_ctx_t *fdctx)
{
+ clnt_conf_t *conf = NULL;
call_frame_t *fr = NULL;
int32_t ret = -1;
+ fd_lk_ctx_t *lk_ctx = NULL;
if (!fdctx)
goto out;
+ conf = (clnt_conf_t *) this->private;
+
if (fdctx->remote_fd == -1) {
gf_log (this->name, GF_LOG_DEBUG, "not a valid fd");
goto out;
}
+ pthread_mutex_lock (&conf->lock);
+ {
+ lk_ctx = fdctx->lk_ctx;
+ fdctx->lk_ctx = NULL;
+ }
+ pthread_mutex_unlock (&conf->lock);
+
+ fd_lk_ctx_unref (lk_ctx);
+
fr = create_frame (this, this->ctx->pool);
if (fdctx->is_dir) {
@@ -4466,7 +4480,6 @@ unwind:
return 0;
}
-
int32_t
client3_1_lk (call_frame_t *frame, xlator_t *this,
void *data)
@@ -4523,6 +4536,7 @@ client3_1_lk (call_frame_t *frame, xlator_t *this,
req.cmd = gf_cmd;
req.type = gf_type;
gf_proto_flock_from_flock (&req.flock, args->flock);
+
memcpy (req.gfid, args->fd->inode->gfid, 16);
ret = client_submit_request (this, &req, frame, conf->fops, GFS3_OP_LK,