posix,shard: Fill in ia_size and ia_blocks before unwinding (f)setattr

The shard xlator will request for size xattrs in its request and posix will perform xattr_fill of requested keys before unwinding (f)setattr. Change-Id: I4ec4779568be6f805724b875b1b4215787112e7f BUG: 1246082 Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com> Reviewed-on: http://review.gluster.org/11754 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com> Tested-by: NetBSD Build System <jenkins@build.gluster.org>
author: Krutika Dhananjay <kdhananj@redhat.com> 2015-07-23 14:59:19 +0530
committer: Pranith Kumar Karampuri <pkarampu@redhat.com> 2015-07-26 23:06:19 -0700
commit: fd296d90168002a475a9899a72a0dd1811f038ee (patch)
tree: 5ca35dc8e73ea0b1722c82215c91a87e5fd70faf /doc/developer-guide
parent: 7bec717f8850135368609fccf1b1c697af60c546 (diff)
0 files changed, 0 insertions, 0 deletions
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index 37a782b31..af01f2ef2 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -51,7 +51,7 @@
 
 #define AFR_ICTX_OPENDIR_DONE_MASK     0x0000000100000000ULL
 #define AFR_ICTX_READ_CHILD_MASK       0x00000000FFFFFFFFULL
-
+#define AFR_STATISTICS_HISTORY_SIZE    50
 int
 afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
                                 gf_boolean_t fail_conflict);
@@ -1080,6 +1080,88 @@ afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent)
                 uuid_copy (loc->pargfid, postparent->ia_gfid);
 }
 
+/*
+ * Quota size xattrs are not maintained by afr. There is a
+ * possibility that they differ even when both the directory changelog xattrs
+ * suggest everything is fine. So if there is at least one 'source' check among
+ * the sources which has the maximum quota size. Otherwise check among all the
+ * available ones for maximum quota size. This way if there is a source and
+ * stale copies it always votes for the 'source'.
+ * */
+
+static void
+afr_handle_quota_size (afr_local_t *local, xlator_t *this,
+                       dict_t *rsp_dict)
+{
+        int32_t       *sources       = NULL;
+        dict_t        *xattr         = NULL;
+        data_t        *max_data      = NULL;
+        int64_t       max_quota_size = -1;
+        data_t        *data          = NULL;
+        int64_t       *size          = NULL;
+        int64_t       quota_size     = -1;
+        afr_private_t *priv          = NULL;
+        int           i              = 0;
+        int           ret            = -1;
+        gf_boolean_t  source_present = _gf_false;
+
+        priv    = this->private;
+        sources = local->cont.lookup.sources;
+
+        if (rsp_dict == NULL) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "%s: Invalid "
+                                  "response dictionary", local->loc.path);
+                return;
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (sources[i]) {
+                        source_present = _gf_true;
+                        break;
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                /*
+                 * If there is at least one source lets check
+                 * for maximum quota sizes among sources, otherwise take the
+                 * maximum of the ones present to be on the safer side.
+                 */
+                if (source_present && !sources[i])
+                        continue;
+
+                xattr = local->cont.lookup.xattrs[i];
+                if (!xattr)
+                        continue;
+
+                data = dict_get (xattr, QUOTA_SIZE_KEY);
+                if (!data)
+                        continue;
+
+                size = (int64_t*)data->data;
+                quota_size = ntoh64(*size);
+                gf_log (this->name, GF_LOG_DEBUG, "%s: %d, size: %"PRId64,
+                        local->loc.path, i, quota_size);
+                if (quota_size > max_quota_size) {
+                        if (max_data)
+                                data_unref (max_data);
+
+                        max_quota_size = quota_size;
+                        max_data = data_ref (data);
+                }
+        }
+
+        if (max_data) {
+                ret = dict_set (rsp_dict, QUOTA_SIZE_KEY, max_data);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s: Failed to set "
+                                "quota size", local->loc.path);
+                }
+
+                data_unref (max_data);
+        }
+}
+
 int
 afr_lookup_build_response_params (afr_local_t *local, xlator_t *this)
 {
@@ -1124,13 +1206,18 @@ afr_lookup_build_response_params (afr_local_t *local, xlator_t *this)
                 ret = -1;
                 goto out;
         }
+
         gf_log (this->name, GF_LOG_DEBUG, "Building lookup response from %d",
                 read_child);
         if (!*xattr)
                 *xattr = dict_ref (local->cont.lookup.xattrs[read_child]);
+
         *buf = local->cont.lookup.bufs[read_child];
         *postparent = local->cont.lookup.postparents[read_child];
 
+        if (dict_get (local->xattr_req, QUOTA_SIZE_KEY))
+                afr_handle_quota_size (local, this, *xattr);
+
         if (IA_INVAL == local->cont.lookup.inode->ia_type) {
                 /* fix for RT #602 */
                 local->cont.lookup.inode->ia_type = buf->ia_type;
@@ -1745,7 +1832,8 @@ afr_lookup_perform_self_heal (call_frame_t *frame, xlator_t *this,
 
         afr_lookup_set_self_heal_params (local, this);
         if (afr_can_self_heal_proceed (&local->self_heal, priv)) {
-                if  (afr_is_transaction_running (local))
+                if  (afr_is_transaction_running (local) &&
+                     (!local->allow_sh_for_running_transaction))
                         goto out;
 
                 reason = "lookup detected pending operations";
@@ -2340,7 +2428,7 @@ afr_lookup (call_frame_t *frame, xlator_t *this,
         int               call_count     = 0;
         uint64_t          ctx            = 0;
         int32_t           op_errno       = 0;
-
+        int               allow_sh       = 0;
         priv = this->private;
 
         AFR_LOCAL_ALLOC_OR_GOTO (local, out);
@@ -2412,6 +2500,11 @@ afr_lookup (call_frame_t *frame, xlator_t *this,
         /* By default assume ENOTCONN. On success it will be set to 0. */
         local->op_errno = ENOTCONN;
 
+        ret = dict_get_int32 (xattr_req, "allow-sh-for-running-transaction",
+                              &allow_sh);
+        dict_del (xattr_req, "allow-sh-for-running-transaction");
+        local->allow_sh_for_running_transaction = allow_sh;
+
         ret = afr_lookup_xattr_req_prepare (local, this, xattr_req, &local->loc,
                                             &gfid_req);
         if (ret) {
@@ -2757,13 +2850,13 @@ afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_ret = 0;
 
                         if (local->success_count == 0) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
+                                local->cont.inode_wfop.prebuf  = *prebuf;
+                                local->cont.inode_wfop.postbuf = *postbuf;
                         }
 
                         if (child_index == read_child) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
+                                local->cont.inode_wfop.prebuf  = *prebuf;
+                                local->cont.inode_wfop.postbuf = *postbuf;
                         }
 
                         local->success_count++;
@@ -2787,8 +2880,9 @@ afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		*/
 		stub = fop_fsync_cbk_stub (frame, afr_fsync_unwind_cbk,
                                            local->op_ret, local->op_errno,
-                                           &local->cont.fsync.prebuf,
-                                           &local->cont.fsync.postbuf, xdata);
+                                           &local->cont.inode_wfop.prebuf,
+                                           &local->cont.inode_wfop.postbuf,
+                                           xdata);
 		if (!stub) {
 			AFR_STACK_UNWIND (fsync, frame, -1, ENOMEM, 0, 0, 0);
 			return 0;
@@ -2860,495 +2954,6 @@ out:
 
 /* }}} */
 
-/* {{{ fallocate */
-
-static int
-afr_fallocate_unwind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (fallocate, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.fallocate.prebuf,
-                                  &local->cont.fallocate.postbuf,
-                                  NULL);
-        }
-        return 0;
-}
-
-static int
-afr_fallocate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
-{
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int child_index = (long) cookie;
-        int call_count  = -1;
-        int need_unwind = 0;
-        int read_child  = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno)) {
-                        afr_transaction_fop_failed (frame, this, child_index);
-                        local->child_errno[child_index] = op_errno;
-                }
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.fallocate.prebuf  = *prebuf;
-                                local->cont.fallocate.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.fallocate.prebuf  = *prebuf;
-                                local->cont.fallocate.postbuf = *postbuf;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
-}
-
-static int
-afr_fallocate_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fallocate_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fallocate,
-                                           local->fd,
-					   local->cont.fallocate.mode,
-                                           local->cont.fallocate.offset,
-					   local->cont.fallocate.len,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
-
-static int
-afr_fallocate_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-static int
-afr_do_fallocate(call_frame_t *frame, xlator_t *this)
-{
-        call_frame_t * transaction_frame = NULL;
-        afr_local_t *  local             = NULL;
-        int op_ret   = -1;
-        int op_errno = 0;
-
-        local = frame->local;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                goto out;
-        }
-
-        transaction_frame->local = local;
-        frame->local = NULL;
-
-        local->op = GF_FOP_FALLOCATE;
-
-        local->transaction.fop    = afr_fallocate_wind;
-        local->transaction.done   = afr_fallocate_done;
-        local->transaction.unwind = afr_fallocate_unwind;
-
-        local->transaction.main_frame = frame;
-
-        local->transaction.start   = local->cont.fallocate.offset;
-        local->transaction.len     = 0;
-
-	/* fallocate can modify the file size */
-        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (op_ret < 0) {
-            op_errno = -op_ret;
-            goto out;
-        }
-
-        op_ret = 0;
-out:
-        if (op_ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fallocate, frame, op_ret, op_errno, NULL,
-                                  NULL, NULL);
-        }
-
-        return 0;
-}
-
-int
-afr_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-	      off_t offset, size_t len, dict_t *xdata)
-{
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv = this->private;
-
-        if (afr_is_split_brain (this, fd->inode)) {
-                op_errno = EIO;
-                goto out;
-        }
-        QUORUM_CHECK(fallocate,out);
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-	local->cont.fallocate.mode = mode;
-        local->cont.fallocate.offset  = offset;
-	local->cont.fallocate.len = len;
-
-        local->fd = fd_ref (fd);
-
-        afr_open_fd_fix (fd, this);
-
-        afr_do_fallocate (frame, this);
-
-        ret = 0;
-out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
-        }
-
-        return 0;
-}
-
-/* }}} */
-
-/* {{{ discard */
-
-static int
-afr_discard_unwind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (discard, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.discard.prebuf,
-                                  &local->cont.discard.postbuf,
-                                  NULL);
-        }
-        return 0;
-}
-
-static int
-afr_discard_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
-{
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int child_index = (long) cookie;
-        int call_count  = -1;
-        int need_unwind = 0;
-        int read_child  = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno)) {
-                        afr_transaction_fop_failed (frame, this, child_index);
-                        local->child_errno[child_index] = op_errno;
-                }
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.discard.prebuf  = *prebuf;
-                                local->cont.discard.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.discard.prebuf  = *prebuf;
-                                local->cont.discard.postbuf = *postbuf;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
-}
-
-static int
-afr_discard_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_discard_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->discard,
-                                           local->fd,
-                                           local->cont.discard.offset,
-					   local->cont.discard.len,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
-
-static int
-afr_discard_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-static int
-afr_do_discard(call_frame_t *frame, xlator_t *this)
-{
-        call_frame_t * transaction_frame = NULL;
-        afr_local_t *  local             = NULL;
-        int op_ret   = -1;
-        int op_errno = 0;
-
-        local = frame->local;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                goto out;
-        }
-
-        transaction_frame->local = local;
-        frame->local = NULL;
-
-        local->op = GF_FOP_DISCARD;
-
-        local->transaction.fop    = afr_discard_wind;
-        local->transaction.done   = afr_discard_done;
-        local->transaction.unwind = afr_discard_unwind;
-
-        local->transaction.main_frame = frame;
-
-        local->transaction.start   = local->cont.discard.offset;
-        local->transaction.len     = 0;
-
-        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (op_ret < 0) {
-            op_errno = -op_ret;
-            goto out;
-        }
-
-        op_ret = 0;
-out:
-        if (op_ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (discard, frame, op_ret, op_errno, NULL,
-                                  NULL, NULL);
-        }
-
-        return 0;
-}
-
-int
-afr_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	    size_t len, dict_t *xdata)
-{
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv = this->private;
-
-        if (afr_is_split_brain (this, fd->inode)) {
-                op_errno = EIO;
-                goto out;
-        }
-        QUORUM_CHECK(discard, out);
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        local->cont.discard.offset  = offset;
-	local->cont.discard.len = len;
-
-        local->fd = fd_ref (fd);
-
-        afr_open_fd_fix (fd, this);
-
-        afr_do_discard(frame, this);
-
-        ret = 0;
-out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
-        }
-
-        return 0;
-}
-
-/* }}} */
-
 /* {{{ fsync */
 
 int32_t
@@ -4826,6 +4431,16 @@ afr_priv_destroy (afr_private_t *priv)
         if (priv->shd.split_brain)
                 eh_destroy (priv->shd.split_brain);
 
+        for (i = 0; i < priv->child_count; i++)
+        {
+                if (priv->shd.statistics[i])
+                        eh_destroy (priv->shd.statistics[i]);
+        }
+
+        GF_FREE (priv->shd.statistics);
+
+        GF_FREE (priv->shd.crawl_events);
+
         GF_FREE (priv->last_event);
         if (priv->pending_key) {
                 for (i = 0; i < priv->child_count; i++)
@@ -4929,3 +4544,48 @@ afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this)
         }
         UNLOCK (&inode->lock);
 }
+
+int
+afr_initialise_statistics (xlator_t *this)
+{
+        afr_private_t       *priv = NULL;
+        int                 ret = -1;
+        int                 i = 0;
+        int                 child_count = 0;
+        eh_t                *stats_per_brick = NULL;
+        shd_crawl_event_t   ***shd_crawl_events = NULL;
+        priv = this->private;
+
+        priv->shd.statistics = GF_CALLOC (sizeof(eh_t *), priv->child_count,
+                                          gf_common_mt_eh_t);
+        if (!priv->shd.statistics) {
+                ret = -1;
+                goto out;
+        }
+        child_count = priv->child_count;
+        for (i=0; i < child_count ; i++) {
+                stats_per_brick = eh_new (AFR_STATISTICS_HISTORY_SIZE,
+                                          _gf_false,
+                                          _destroy_crawl_event_data);
+                if (!stats_per_brick) {
+                        ret = -1;
+                        goto out;
+                }
+                priv->shd.statistics[i] = stats_per_brick;
+
+        }
+
+        shd_crawl_events = (shd_crawl_event_t***)(&priv->shd.crawl_events);
+        *shd_crawl_events  = GF_CALLOC (sizeof(shd_crawl_event_t*),
+                                        priv->child_count,
+                                        gf_afr_mt_shd_crawl_event_t);
+
+        if (!priv->shd.crawl_events) {
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+
+}
diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index 00dd22214..e06e3b2f2 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -117,6 +117,8 @@ afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
 
         children = priv->children;
 
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
         AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
@@ -232,6 +234,8 @@ afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 
         children = priv->children;
 
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
         AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
@@ -348,10 +352,7 @@ afr_fstat (call_frame_t *frame, xlator_t *this,
 
         VALIDATE_OR_GOTO (fd->inode, out);
 
-        if (afr_is_split_brain (this, fd->inode)) {
-                op_errno = EIO;
-                goto out;
-        }
+        AFR_SBRAIN_CHECK_FD (fd, out);
 
         AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
@@ -472,6 +473,8 @@ afr_readlink (call_frame_t *frame, xlator_t *this,
 
         children = priv->children;
 
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
         AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
@@ -1472,6 +1475,8 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
 
         children = priv->children;
 
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
         AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
 
@@ -1727,10 +1732,8 @@ afr_fgetxattr (call_frame_t *frame, xlator_t *this,
 
         children = priv->children;
 
-        if (afr_is_split_brain (this, fd->inode)) {
-                op_errno = EIO;
-                goto out;
-        }
+        AFR_SBRAIN_CHECK_FD (fd, out);
+
         AFR_LOCAL_ALLOC_OR_GOTO (local, out);
         frame->local = local;
 
@@ -1886,10 +1889,7 @@ afr_readv (call_frame_t *frame, xlator_t *this,
         priv     = this->private;
         children = priv->children;
 
-        if (afr_is_split_brain (this, fd->inode)) {
-                op_errno = EIO;
-                goto out;
-        }
+        AFR_SBRAIN_CHECK_FD (fd, out);
 
         AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
         local = frame->local;
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index a3090e4db..c1ec69a55 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -39,6 +39,46 @@
 #include "afr-transaction.h"
 #include "afr-self-heal-common.h"
 
+void
+__inode_write_fop_cbk (call_frame_t *frame, int child_index, int read_child,
+                       xlator_t *this, int32_t *op_ret, int32_t *op_errno,
+                       struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t     *local = NULL;
+
+        local = frame->local;
+
+        if (afr_fop_failed (*op_ret, *op_errno)) {
+                local->child_errno[child_index] = *op_errno;
+
+                switch (local->op) {
+                case GF_FOP_TRUNCATE:
+                case GF_FOP_FTRUNCATE:
+                        if (*op_errno != EFBIG)
+                                afr_transaction_fop_failed (frame, this,
+                                                            child_index);
+                break;
+                default:
+                        afr_transaction_fop_failed (frame, this, child_index);
+                break;
+                }
+                local->op_errno = *op_errno;
+                goto out;
+        }
+
+        if ((local->success_count == 0) || (read_child == child_index)) {
+                local->op_ret              = *op_ret;
+                if (prebuf)
+                        local->cont.inode_wfop.prebuf  = *prebuf;
+                if (postbuf)
+                        local->cont.inode_wfop.postbuf = *postbuf;
+        }
+
+        local->success_count++;
+out:
+        return;
+}
+
 /* {{{ writev */
 
 void
@@ -52,8 +92,8 @@ afr_writev_copy_outvars (call_frame_t *src_frame, call_frame_t *dst_frame)
 
         dst_local->op_ret = src_local->op_ret;
         dst_local->op_errno = src_local->op_errno;
-        dst_local->cont.writev.prebuf = src_local->cont.writev.prebuf;
-        dst_local->cont.writev.postbuf = src_local->cont.writev.postbuf;
+        dst_local->cont.inode_wfop.prebuf = src_local->cont.inode_wfop.prebuf;
+        dst_local->cont.inode_wfop.postbuf = src_local->cont.inode_wfop.postbuf;
 }
 
 void
@@ -64,8 +104,8 @@ afr_writev_unwind (call_frame_t *frame, xlator_t *this)
 
         AFR_STACK_UNWIND (writev, frame,
                           local->op_ret, local->op_errno,
-                          &local->cont.writev.prebuf,
-                          &local->cont.writev.postbuf,
+                          &local->cont.inode_wfop.prebuf,
+                          &local->cont.inode_wfop.postbuf,
                           NULL);
 }
 
@@ -153,14 +193,14 @@ afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->read_child_returned = _gf_true;
                 }
 
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
+
 		local->replies[child_index].valid = 1;
 		local->replies[child_index].op_ret = op_ret;
 		local->replies[child_index].op_errno = op_errno;
 
-                if (afr_fop_failed (op_ret, op_errno)) {
-                        afr_transaction_fop_failed (frame, this, child_index);
-                        local->child_errno[child_index] = op_errno;
-                }
 
 		/* stage the best case return value for unwind */
                 if ((local->success_count == 0) || (op_ret > local->op_ret)) {
@@ -187,12 +227,6 @@ afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 					local->append_write = _gf_false;
                         }
 
-			if ((local->success_count == 0) ||
-			    (child_index == read_child)) {
-				local->cont.writev.prebuf  = *prebuf;
-				local->cont.writev.postbuf = *postbuf;
-			}
-			local->success_count++;
 		}
         }
         UNLOCK (&frame->lock);
@@ -572,8 +606,8 @@ afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (truncate, main_frame, local->op_ret,
                                   local->op_errno,
-                                  &local->cont.truncate.prebuf,
-                                  &local->cont.truncate.postbuf,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
                                   NULL);
         }
 
@@ -601,29 +635,13 @@ afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->read_child_returned = _gf_true;
                 }
 
-                if (afr_fop_failed (op_ret, op_errno) && op_errno != EFBIG) {
-                        afr_transaction_fop_failed (frame, this, child_index);
-                        local->child_errno[child_index] = op_errno;
-                }
-
                 if (op_ret != -1) {
 			if (prebuf->ia_size != postbuf->ia_size)
 				local->stable_write = _gf_false;
-
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
-
-                        local->success_count++;
                 }
-                local->op_errno = op_errno;
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
         }
         UNLOCK (&frame->lock);
 
@@ -781,8 +799,8 @@ afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (ftruncate, main_frame, local->op_ret,
                                   local->op_errno,
-                                  &local->cont.ftruncate.prebuf,
-                                  &local->cont.ftruncate.postbuf,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
                                   NULL);
         }
         return 0;
@@ -809,29 +827,13 @@ afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->read_child_returned = _gf_true;
                 }
 
-                if (afr_fop_failed (op_ret, op_errno)) {
-                        afr_transaction_fop_failed (frame, this, child_index);
-                        local->child_errno[child_index] = op_errno;
-                }
-
                 if (op_ret != -1) {
 			if (prebuf->ia_size != postbuf->ia_size)
 				local->stable_write = _gf_false;
-
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
-
-                        local->success_count++;
                 }
-                local->op_errno = op_errno;
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
         }
         UNLOCK (&frame->lock);
 
@@ -1023,8 +1025,8 @@ afr_setattr_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (setattr, main_frame, local->op_ret,
                                   local->op_errno,
-                                  &local->cont.setattr.preop_buf,
-                                  &local->cont.setattr.postop_buf,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
                                   NULL);
         }
 
@@ -1055,29 +1057,14 @@ afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->read_child_returned = _gf_true;
                 }
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
-
-                        local->success_count++;
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, preop, postop,
+                                       xdata);
 
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
                 }
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -1232,8 +1219,8 @@ afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
         if (main_frame) {
                 AFR_STACK_UNWIND (fsetattr, main_frame, local->op_ret,
                                   local->op_errno,
-                                  &local->cont.fsetattr.preop_buf,
-                                  &local->cont.fsetattr.postop_buf,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
                                   NULL);
         }
 
@@ -1264,29 +1251,14 @@ afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->read_child_returned = _gf_true;
                 }
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, preop, postop,
+                                       xdata);
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
                 }
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -1459,28 +1431,23 @@ int
 afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t      *local         = NULL;
-        afr_private_t    *priv          = NULL;
-        int               call_count    = -1;
-        int               need_unwind   = 0;
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        int           call_count  = -1;
+        int           need_unwind = 0;
+        int           child_index = (long) cookie;
 
         local = frame->local;
         priv = this->private;
 
         LOCK (&frame->lock);
         {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->child_count) {
-                                need_unwind = 1;
-                        }
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
+                if (local->success_count == priv->child_count) {
+                        need_unwind = 1;
                 }
-
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -1649,28 +1616,24 @@ int
 afr_fsetxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t     *local       = NULL;
-        afr_private_t   *priv        = NULL;
-        int              call_count  = -1;
-        int              need_unwind = 0;
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        int           call_count  = -1;
+        int           need_unwind = 0;
+        int           child_index = (long) cookie;
 
         local = frame->local;
         priv = this->private;
 
         LOCK (&frame->lock);
         {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
 
-                        if (local->success_count == priv->child_count) {
-                                need_unwind = 1;
-                        }
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
+                if (local->success_count == priv->child_count) {
+                        need_unwind = 1;
                 }
-
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -1849,28 +1812,23 @@ int
 afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int call_count  = -1;
-        int need_unwind = 0;
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        int           call_count  = -1;
+        int           need_unwind = 0;
+        int           child_index = (long) cookie;
 
         local = frame->local;
         priv = this->private;
 
         LOCK (&frame->lock);
         {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
+                if (local->success_count == priv->wait_count) {
+                        need_unwind = 1;
                 }
-
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -2038,28 +1996,24 @@ int
 afr_fremovexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int call_count  = -1;
-        int need_unwind = 0;
+        afr_local_t *   local       = NULL;
+        afr_private_t * priv        = NULL;
+        int             call_count  = -1;
+        int             need_unwind = 0;
+        int             child_index = (long) cookie;
 
         local = frame->local;
         priv = this->private;
 
         LOCK (&frame->lock);
         {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
 
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
+                if (local->success_count == priv->wait_count) {
+                        need_unwind = 1;
                 }
-
-                local->op_errno = op_errno;
         }
         UNLOCK (&frame->lock);
 
@@ -2204,3 +2158,704 @@ out:
 
         return 0;
 }
+
+static int
+afr_fallocate_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (fallocate, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
+        }
+        return 0;
+}
+
+static int
+afr_fallocate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = NULL;
+        int child_index = (long) cookie;
+        int call_count  = -1;
+        int need_unwind = 0;
+        int read_child  = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
+
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
+
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (need_unwind)
+                local->transaction.unwind (frame, this);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
+
+        return 0;
+}
+
+static int
+afr_fallocate_wind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_fallocate_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->fallocate,
+                                           local->fd,
+                                           local->cont.fallocate.mode,
+                                           local->cont.fallocate.offset,
+                                           local->cont.fallocate.len,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
+
+static int
+afr_fallocate_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+
+        local = frame->local;
+
+        local->transaction.unwind (frame, this);
+
+        AFR_STACK_DESTROY (frame);
+
+        return 0;
+}
+
+static int
+afr_do_fallocate (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t * transaction_frame = NULL;
+        afr_local_t *  local             = NULL;
+        int op_ret   = -1;
+        int op_errno = 0;
+
+        local = frame->local;
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                goto out;
+        }
+
+        transaction_frame->local = local;
+        frame->local = NULL;
+
+        local->op = GF_FOP_FALLOCATE;
+
+        local->transaction.fop    = afr_fallocate_wind;
+        local->transaction.done   = afr_fallocate_done;
+        local->transaction.unwind = afr_fallocate_unwind;
+
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.fallocate.offset;
+        local->transaction.len     = 0;
+
+        /* fallocate can modify the file size */
+        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
+
+        op_ret = 0;
+out:
+        if (op_ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (fallocate, frame, op_ret, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
+
+int
+afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+               off_t offset, size_t len, dict_t *xdata)
+{
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        call_frame_t   *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+        QUORUM_CHECK(fallocate,out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->cont.fallocate.mode = mode;
+        local->cont.fallocate.offset  = offset;
+        local->cont.fallocate.len = len;
+
+        local->fd = fd_ref (fd);
+
+        afr_open_fd_fix (fd, this);
+
+        afr_do_fallocate (frame, this);
+
+        ret = 0;
+out:
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        }
+
+        return 0;
+}
+
+/* }}} */
+
+/* {{{ discard */
+
+static int
+afr_discard_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (discard, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
+        }
+        return 0;
+}
+
+static int
+afr_discard_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = NULL;
+        int child_index = (long) cookie;
+        int call_count  = -1;
+        int need_unwind = 0;
+        int read_child  = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
+
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
+
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (need_unwind)
+                local->transaction.unwind (frame, this);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
+
+        return 0;
+}
+
+static int
+afr_discard_wind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_discard_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->discard,
+                                           local->fd,
+                                           local->cont.discard.offset,
+                                           local->cont.discard.len,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
+
+static int
+afr_discard_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+
+        local = frame->local;
+
+        local->transaction.unwind (frame, this);
+
+        AFR_STACK_DESTROY (frame);
+
+        return 0;
+}
+
+static int
+afr_do_discard (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t * transaction_frame = NULL;
+        afr_local_t *  local             = NULL;
+        int op_ret   = -1;
+        int op_errno = 0;
+
+        local = frame->local;
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                goto out;
+        }
+
+        transaction_frame->local = local;
+        frame->local = NULL;
+
+        local->op = GF_FOP_DISCARD;
+
+        local->transaction.fop    = afr_discard_wind;
+        local->transaction.done   = afr_discard_done;
+        local->transaction.unwind = afr_discard_unwind;
+
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.discard.offset;
+        local->transaction.len     = 0;
+
+        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
+
+        op_ret = 0;
+out:
+        if (op_ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (discard, frame, op_ret, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
+
+int
+afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata)
+{
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        call_frame_t   *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+        QUORUM_CHECK(discard, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->cont.discard.offset  = offset;
+        local->cont.discard.len = len;
+
+        local->fd = fd_ref (fd);
+
+        afr_open_fd_fix (fd, this);
+
+        afr_do_discard(frame, this);
+
+        ret = 0;
+out:
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+        }
+
+        return 0;
+}
+
+
+/* {{{ zerofill */
+
+static int
+afr_zerofill_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local            = NULL;
+        call_frame_t    *main_frame       = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame) {
+                        main_frame = local->transaction.main_frame;
+                }
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (zerofill, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.zerofill.prebuf,
+                                  &local->cont.zerofill.postbuf,
+                                  NULL);
+        }
+        return 0;
+}
+
+static int
+afr_zerofill_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t       *local             = NULL;
+        afr_private_t     *priv              = NULL;
+        int                child_index       = (long) cookie;
+        int                call_count        = -1;
+        int                need_unwind       = 0;
+        int                read_child        = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
+
+                if (afr_fop_failed (op_ret, op_errno)) {
+                        afr_transaction_fop_failed (frame, this, child_index);
+                }
+
+                if (op_ret != -1) {
+                        if (local->success_count == 0) {
+                                local->op_ret = op_ret;
+                                local->cont.zerofill.prebuf  = *prebuf;
+                                local->cont.zerofill.postbuf = *postbuf;
+                        }
+
+                        if (child_index == read_child) {
+                                local->cont.zerofill.prebuf  = *prebuf;
+                                local->cont.zerofill.postbuf = *postbuf;
+                        }
+
+                        local->success_count++;
+
+                        if ((local->success_count >= priv->wait_count)
+                            && local->read_child_returned) {
+                                need_unwind = 1;
+                        }
+                }
+                local->op_errno = op_errno;
+        }
+        UNLOCK (&frame->lock);
+
+        if (need_unwind) {
+                local->transaction.unwind (frame, this);
+        }
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
+
+        return 0;
+}
+
+static int
+afr_zerofill_wind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t    *local         = NULL;
+        afr_private_t  *priv          = NULL;
+        int             call_count    = -1;
+        int             i             = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_zerofill_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->zerofill,
+                                           local->fd,
+                                           local->cont.zerofill.offset,
+                                           local->cont.zerofill.len,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
+
+static int
+afr_zerofill_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+
+        local = frame->local;
+
+        local->transaction.unwind (frame, this);
+
+        AFR_STACK_DESTROY (frame);
+
+        return 0;
+}
+
+static int
+afr_do_zerofill(call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t  *transaction_frame = NULL;
+        afr_local_t   *local             = NULL;
+        int            op_ret            = -1;
+        int            op_errno          = 0;
+
+        local = frame->local;
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                goto out;
+        }
+
+        transaction_frame->local = local;
+        frame->local = NULL;
+
+        local->op = GF_FOP_ZEROFILL;
+
+        local->transaction.fop    = afr_zerofill_wind;
+        local->transaction.done   = afr_zerofill_done;
+        local->transaction.unwind = afr_zerofill_unwind;
+
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.zerofill.offset;
+        local->transaction.len     = 0;
+
+        op_ret = afr_transaction (transaction_frame, this,
+                                  AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+                op_errno = -op_ret;
+                goto out;
+        }
+
+        op_ret = 0;
+out:
+        if (op_ret < 0) {
+                if (transaction_frame) {
+                        AFR_STACK_DESTROY (transaction_frame);
+                }
+                AFR_STACK_UNWIND (zerofill, frame, op_ret, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
+
+int
+afr_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              size_t len, dict_t *xdata)
+{
+        afr_private_t   *priv               = NULL;
+        afr_local_t     *local              = NULL;
+        call_frame_t    *transaction_frame  = NULL;
+        int              ret                = -1;
+        int              op_errno           = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+        QUORUM_CHECK(zerofill, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0) {
+                goto out;
+        }
+        local->cont.zerofill.offset  = offset;
+        local->cont.zerofill.len = len;
+
+        local->fd = fd_ref (fd);
+
+        afr_open_fd_fix (fd, this);
+
+        afr_do_zerofill(frame, this);
+
+        ret = 0;
+out:
+        if (ret < 0) {
+                if (transaction_frame) {
+                        AFR_STACK_DESTROY (transaction_frame);
+                }
+                AFR_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
+
+/* }}} */
+
+
diff --git a/xlators/cluster/afr/src/afr-inode-write.h b/xlators/cluster/afr/src/afr-inode-write.h
index ed11079fd..8e93ca44a 100644
--- a/xlators/cluster/afr/src/afr-inode-write.h
+++ b/xlators/cluster/afr/src/afr-inode-write.h
@@ -68,4 +68,15 @@ int32_t
 afr_fremovexattr (call_frame_t *frame, xlator_t *this,
                   fd_t *fd, const char *name, dict_t *xdata);
 
+int
+afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata);
+
+int
+afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+               off_t offset, size_t len, dict_t *xdata);
+
+int
+afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata);
 #endif /* __INODE_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-lk-common.c b/xlators/cluster/afr/src/afr-lk-common.c
index e2136089b..060d78f35 100644
--- a/xlators/cluster/afr/src/afr-lk-common.c
+++ b/xlators/cluster/afr/src/afr-lk-common.c
@@ -658,6 +658,7 @@ afr_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         afr_internal_lock_t *int_lock = NULL;
         afr_inodelk_t       *inodelk = NULL;
         int32_t             child_index = (long)cookie;
+        afr_private_t       *priv = NULL;
 
         local = frame->local;
         int_lock = &local->internal_lock;
@@ -666,9 +667,12 @@ afr_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                AFR_UNLOCK_OP, NULL, op_ret,
                                op_errno, child_index);
 
+        priv = this->private;
+
         if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
-                gf_log (this->name, GF_LOG_INFO, "%s: unlock failed on %d "
-                        "unlock by %s", local->loc.path, child_index,
+                gf_log (this->name, GF_LOG_INFO, "%s: unlock failed on subvolume %s "
+                        "with lock owner %s", local->loc.path,
+                        priv->children[child_index]->name,
                         lkowner_utoa (&frame->root->lk_owner));
         }
 
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
index e01ab366f..73594f265 100644
--- a/xlators/cluster/afr/src/afr-mem-types.h
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -41,7 +41,10 @@ enum gf_afr_mem_types_ {
         gf_afr_mt_shd_event_t,
         gf_afr_mt_time_t,
         gf_afr_mt_pos_data_t,
-	gf_afr_mt_reply_t,
+        gf_afr_mt_reply_t,
+        gf_afr_mt_stats_t,
+        gf_afr_mt_shd_crawl_event_t,
+        gf_afr_mt_uint64_t,
         gf_afr_mt_end
 };
 #endif
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
index f33b04eed..1b48a1bca 100644
--- a/xlators/cluster/afr/src/afr-self-heald.c
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -25,7 +25,8 @@ typedef enum {
 
 typedef enum {
         HEAL = 1,
-        INFO
+        INFO,
+        STATISTICS_TO_BE_HEALED,
 } shd_crawl_op;
 
 typedef struct shd_dump {
@@ -84,6 +85,33 @@ _loc_assign_gfid_path (loc_t *loc)
 }
 
 void
+_destroy_crawl_event_data (void *data)
+{
+        shd_crawl_event_t        *crawl_event = NULL;
+
+        if (!data)
+                goto out;
+
+        crawl_event = (shd_crawl_event_t *)data;
+        GF_FREE (crawl_event->start_time_str);
+        GF_FREE (crawl_event->end_time_str);
+
+out:
+        return;
+}
+
+void
+_destroy_shd_event_data (void *data)
+{
+        shd_event_t             *event = NULL;
+        if (!data)
+                goto out;
+        event = (shd_event_t*)data;
+        GF_FREE (event->path);
+out:
+        return;
+}
+void
 shd_cleanup_event (void *event)
 {
         shd_event_t *shd_event = event;
@@ -128,6 +156,123 @@ _build_index_loc (xlator_t *this, loc_t *loc, char *name, loc_t *parent)
 }
 
 int
+_add_crawl_stats_to_dict (xlator_t *this, dict_t *output, int child,
+                          shd_crawl_event_t *shd_event, struct timeval *tv)
+{
+        int             ret = 0;
+        uint64_t        count = 0;
+        char            key[256] = {0};
+        int             xl_id = 0;
+        uint64_t        healed_count = 0;
+        uint64_t        split_brain_count = 0;
+        uint64_t        heal_failed_count = 0;
+        char            *start_time_str = NULL;
+        char            *end_time_str = NULL;
+        char            *crawl_type = NULL;
+        int             progress = -1;
+
+        healed_count = shd_event->healed_count;
+        split_brain_count = shd_event->split_brain_count;
+        heal_failed_count = shd_event->heal_failed_count;
+        start_time_str = shd_event->start_time_str;
+        end_time_str = shd_event->end_time_str;
+        crawl_type = shd_event->crawl_type;
+
+        if (!start_time_str) {
+                ret = -1;
+                goto out;
+        }
+
+
+        ret = dict_get_int32 (output, this->name, &xl_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
+        ret = dict_get_uint64 (output, key, &count);
+
+        snprintf (key, sizeof (key), "statistics_healed_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64(output, key, healed_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "healed_count to outout");
+                goto out;
+         }
+        snprintf (key, sizeof (key), "statistics_sb_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64 (output, key, split_brain_count);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "split_brain_count to outout");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_crawl_type-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_dynstr (output, key, gf_strdup (crawl_type));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "crawl_type to output");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_heal_failed_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64 (output, key, heal_failed_count);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "healed_failed_count to outout");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_strt_time-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_dynstr (output, key, gf_strdup(start_time_str));
+
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "crawl_start_time to outout");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "statistics_end_time-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+
+        if (!end_time_str)
+                end_time_str = "Could not determine the end time";
+        ret = dict_set_dynstr (output, key, gf_strdup(end_time_str));
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "crawl_end_time to outout");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_inprogress-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+
+        if (shd_event->crawl_inprogress == _gf_true)
+                progress = 1;
+        else
+                progress = 0;
+
+        ret = dict_set_int32 (output, key, progress);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "inprogress to outout");
+                goto out;
+        }
+
+         snprintf (key, sizeof (key), "statistics-%d-%d-count",xl_id, child);
+         ret = dict_set_uint64 (output, key, count + 1);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not increment the "
+                        "counter.");
+                goto out;
+         }
+out:
+        return ret;
+}
+
+int
 _add_path_to_dict (xlator_t *this, dict_t *output, int child, char *path,
                    struct timeval *tv, gf_boolean_t dyn)
 {
@@ -233,6 +378,20 @@ out:
 }
 
 int
+_add_crawl_event_statistics_to_dict (circular_buffer_t *cb, void *data)
+{
+        int               ret = 0;
+        shd_dump_t        *dump_data = NULL;
+        shd_crawl_event_t *shd_event = NULL;
+
+        dump_data = data;
+        shd_event = cb->data;
+        ret = _add_crawl_stats_to_dict (dump_data->this, dump_data->dict,
+                                        dump_data->child, shd_event, &cb->tv);
+        return ret;
+}
+
+int
 _add_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict, int child)
 {
         shd_dump_t dump_data = {0};
@@ -244,6 +403,26 @@ _add_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict, int child)
         return 0;
 }
 
+
+int
+_add_statistics_to_dict (xlator_t *this, dict_t *dict, int child)
+{
+        shd_dump_t              dump_data = {0};
+        afr_private_t           *priv  = NULL;
+        afr_self_heald_t        *shd = NULL;
+
+        priv = this->private;
+        shd = &priv->shd;
+
+        dump_data.this = this;
+        dump_data.dict = dict;
+        dump_data.child = child;
+        eh_dump (shd->statistics[child], &dump_data,
+                 _add_crawl_event_statistics_to_dict);
+        return 0;
+
+}
+
 void
 _remove_stale_index (xlator_t *this, xlator_t *readdir_xl,
                      loc_t *parent, char *fname)
@@ -269,6 +448,46 @@ out:
 }
 
 int
+_count_hard_links_under_base_indices_dir (xlator_t *this,
+                                           afr_crawl_data_t *crawl_data,
+                                           gf_dirent_t *entry, loc_t *childloc,
+                                           loc_t *parentloc, struct iatt *iattr)
+{
+        xlator_t                *readdir_xl = crawl_data->readdir_xl;
+        struct iatt             parent = {0};
+        int                     ret = 0;
+        dict_t                  *output = NULL;
+        int                     xl_id =  0;
+        char                    key[256] = {0};
+        int                     child  = -1;
+        uint64_t                     hardlinks = 0;
+
+        output = crawl_data->op_data;
+        child = crawl_data->child;
+
+        ret = syncop_lookup (readdir_xl, childloc, NULL, iattr, NULL, &parent);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (output, this->name, &xl_id);
+        if (ret)
+                goto out;
+
+        snprintf (key, sizeof (key), "%d-%d-hardlinks", xl_id, child);
+        ret =  dict_get_uint64 (output, key, &hardlinks);
+
+        /*Removing the count of base_entry under indices/base_indicies and
+         * entry under indices/xattrop */
+        hardlinks = hardlinks + iattr->ia_nlink - 2;
+        ret = dict_set_uint64 (output, key, hardlinks);
+        if (ret)
+                goto out;
+
+out:
+        return ret;
+}
+
+int
 _add_summary_to_dict (xlator_t *this, afr_crawl_data_t *crawl_data,
                       gf_dirent_t *entry,
                       loc_t *childloc, loc_t *parentloc, struct iatt *iattr)
@@ -307,16 +526,18 @@ _crawl_post_sh_action (xlator_t *this, loc_t *parent, loc_t *child,
                        int32_t op_ret, int32_t op_errno, dict_t *xattr_rsp,
                        afr_crawl_data_t *crawl_data)
 {
-        int              ret = 0;
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-        eh_t             *eh = NULL;
-        char             *path = NULL;
-        char             gfid_str[64] = {0};
-        shd_event_t      *event = NULL;
-        int32_t          sh_failed = 0;
-        gf_boolean_t     split_brain = 0;
-        int32_t          actual_sh_done = 0;
+        int                ret = 0;
+        afr_private_t      *priv = NULL;
+        afr_self_heald_t   *shd = NULL;
+        eh_t               *eh = NULL;
+        char               *path = NULL;
+        char               gfid_str[64] = {0};
+        shd_event_t        *event = NULL;
+        int32_t            sh_failed = 0;
+        gf_boolean_t       split_brain = 0;
+        int32_t            actual_sh_done = 0;
+        shd_crawl_event_t  **shd_crawl_event = NULL;
+
         priv = this->private;
         shd  = &priv->shd;
         if (crawl_data->crawl == INDEX) {
@@ -343,16 +564,19 @@ _crawl_post_sh_action (xlator_t *this, loc_t *parent, loc_t *child,
                 ret = dict_get_int32 (xattr_rsp, "actual-sh-done", &actual_sh_done);
         }
 
-        split_brain = afr_is_split_brain (this, child->inode);
+        shd_crawl_event = (shd_crawl_event_t**)(shd->crawl_events);
 
+        split_brain = afr_is_split_brain (this, child->inode);
         if ((op_ret < 0 && op_errno == EIO) || split_brain) {
                 eh = shd->split_brain;
+                shd_crawl_event[crawl_data->child]->split_brain_count += 1;
         } else if ((op_ret < 0) || sh_failed) {
                 eh = shd->heal_failed;
+                shd_crawl_event[crawl_data->child]->heal_failed_count += 1;
         } else if (actual_sh_done == 1) {
-                      eh = shd->healed;
+                eh = shd->healed;
+                shd_crawl_event[crawl_data->child]->healed_count += 1;
         }
-
         ret = -1;
 
         if (eh != NULL) {
@@ -408,10 +632,20 @@ _self_heal_entry (xlator_t *this, afr_crawl_data_t *crawl_data, gf_dirent_t *ent
         struct iatt      parentbuf = {0};
         int              ret = 0;
         dict_t           *xattr_rsp = NULL;
+        dict_t           *xattr_req = NULL;
+
+        xattr_req = dict_new ();
+        if (!xattr_req) {
+                errno = ENOMEM;
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_int32 (xattr_req, "allow-sh-for-running-transaction", 1);
 
         gf_log (this->name, GF_LOG_DEBUG, "lookup %s", child->path);
 
-        ret = syncop_lookup (this, child, NULL,
+        ret = syncop_lookup (this, child, xattr_req,
                              iattr, &xattr_rsp, &parentbuf);
         _crawl_post_sh_action (this, parent, child, ret, errno, xattr_rsp,
                                crawl_data);
@@ -420,6 +654,9 @@ _self_heal_entry (xlator_t *this, afr_crawl_data_t *crawl_data, gf_dirent_t *ent
         if (ret == 0)
                 ret = _link_inode_update_loc (this, child, iattr);
 
+out:
+        if (xattr_req)
+                dict_unref(xattr_req);
         return ret;
 }
 
@@ -528,12 +765,20 @@ _do_crawl_op_on_local_subvols (xlator_t *this, afr_crawl_type_t crawl,
                                         status = "Started self-heal";
                                         _do_self_heal_on_subvol (this, i,
                                                                  crawl);
-                                } else if (output) {
+                                } else if (output && (op == INFO)) {
                                         status = "";
                                         afr_start_crawl (this, i, INDEX,
                                                          _add_summary_to_dict,
                                                          output, _gf_false, 0,
                                                          NULL);
+                                } else if (output &&
+                                           (op == STATISTICS_TO_BE_HEALED)) {
+                                            status = "";
+                                            afr_start_crawl (this, i,
+                                                             INDEX_TO_BE_HEALED,
+                                       _count_hard_links_under_base_indices_dir,
+                                                             output, _gf_false,
+                                                             0, NULL);
                                 }
                         }
                         if (output) {
@@ -567,8 +812,103 @@ _get_index_summary_on_local_subvols (xlator_t *this, dict_t *output)
         return _do_crawl_op_on_local_subvols (this, INDEX, INFO, output);
 }
 
+void
+afr_fill_completed_crawl_statistics_to_dict (xlator_t *this, dict_t *dict)
+{
+        afr_private_t           *priv  = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     i = 0;
+        priv = this->private;
+        shd= &priv->shd;
+        for (i = 0; i < priv->child_count; i++) {
+                if (shd->pos[i] != AFR_POS_LOCAL)
+                        continue;
+                _add_statistics_to_dict (this, dict, i);
+        }
+
+        return ;
+}
+
+static void
+reset_crawl_event (shd_crawl_event_t *crawl_event)
+{
+    crawl_event->healed_count = 0;
+    crawl_event->split_brain_count = 0;
+    crawl_event->heal_failed_count = 0;
+    GF_FREE (crawl_event->start_time_str);
+    crawl_event->start_time_str = NULL;
+    crawl_event->end_time_str = NULL;
+    crawl_event->crawl_type = NULL;
+    crawl_event->crawl_inprogress = _gf_false;
+    return;
+}
+
+static void
+afr_copy_crawl_event_struct (shd_crawl_event_t *src, shd_crawl_event_t *dst)
+{
+        dst->healed_count = src->healed_count;
+        dst->split_brain_count = src->split_brain_count;
+        dst->heal_failed_count = src->heal_failed_count;
+        dst->start_time_str = gf_strdup (src->start_time_str);
+        dst->end_time_str = "Crawl is already in progress";
+        dst->crawl_type = src->crawl_type;
+        dst->crawl_inprogress = _gf_true;
+        return;
+}
+
+static int
+afr_fill_crawl_statistics_of_running_crawl(xlator_t *this, dict_t *dict)
+{
+        shd_crawl_event_t       *evnt = NULL;
+        int                     ret = 0;
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     i = 0;
+        priv = this->private;
+        shd = &priv->shd;
+
+        evnt = GF_CALLOC (1, sizeof (shd_crawl_event_t),
+                          gf_afr_mt_shd_crawl_event_t);
+        if (!evnt) {
+                ret = -1;
+                goto out;
+        }
+        LOCK (&priv->lock);
+        {
+                for (i = 0; i < priv->child_count; i++) {
+                        if (shd->pos[i] != AFR_POS_LOCAL)
+                                continue;
+
+                        reset_crawl_event (evnt);
+
+                        if (!shd->crawl_events[i]) {
+                                continue;
+                        }
+
+                        afr_copy_crawl_event_struct (shd->crawl_events[i],
+                                                     evnt);
+                        _add_crawl_stats_to_dict (this, dict, i, evnt, NULL);
+
+                }
+        }
+        UNLOCK (&priv->lock);
+        reset_crawl_event (evnt);
+        GF_FREE (evnt);
+
+out:
+        return ret;
+}
+
+static int
+_add_local_subvols_crawl_statistics_to_dict (xlator_t *this, dict_t *dict)
+{
+        int ret = 0;
+        afr_fill_completed_crawl_statistics_to_dict (this, dict);
+        ret = afr_fill_crawl_statistics_of_running_crawl (this, dict);
+        return ret;
+}
 int
-_add_all_subvols_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict)
+_add_local_subvols_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict)
 {
         afr_private_t           *priv = NULL;
         afr_self_heald_t        *shd = NULL;
@@ -618,16 +958,25 @@ afr_xl_op (xlator_t *this, dict_t *input, dict_t *output)
                 ret = 0;
                 break;
         case GF_AFR_OP_HEALED_FILES:
-                ret = _add_all_subvols_eh_to_dict (this, shd->healed, output);
+                ret = _add_local_subvols_eh_to_dict (this, shd->healed, output);
                 break;
         case GF_AFR_OP_HEAL_FAILED_FILES:
-                ret = _add_all_subvols_eh_to_dict (this, shd->heal_failed,
+                ret = _add_local_subvols_eh_to_dict (this, shd->heal_failed,
                                                    output);
                 break;
         case GF_AFR_OP_SPLIT_BRAIN_FILES:
-                ret = _add_all_subvols_eh_to_dict (this, shd->split_brain,
+                ret = _add_local_subvols_eh_to_dict (this, shd->split_brain,
                                                    output);
                 break;
+        case GF_AFR_OP_STATISTICS:
+                ret = _add_local_subvols_crawl_statistics_to_dict (this, output);
+                break;
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                ret = _do_crawl_op_on_local_subvols (this, INDEX_TO_BE_HEALED,
+                                                     STATISTICS_TO_BE_HEALED,
+                                                     output);
+                break;
         default:
                 gf_log (this->name, GF_LOG_ERROR, "Unknown set op %d", op);
                 break;
@@ -642,7 +991,7 @@ afr_poll_self_heal (void *data)
 {
         afr_private_t    *priv = NULL;
         afr_self_heald_t *shd = NULL;
-        struct timeval   timeout = {0};
+        struct timespec  timeout = {0};
         xlator_t         *this = NULL;
         long             child = (long)data;
         gf_timer_t       *old_timer = NULL;
@@ -666,7 +1015,7 @@ afr_poll_self_heal (void *data)
         if (shd->enabled && (shd->pos[child] == AFR_POS_LOCAL))
                 _do_self_heal_on_subvol (this, child, INDEX);
         timeout.tv_sec = shd->timeout;
-        timeout.tv_usec = 0;
+        timeout.tv_nsec = 0;
         //notify and previous timer should be synchronized.
         LOCK (&priv->lock);
         {
@@ -800,6 +1149,7 @@ afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data,
         afr_private_t *priv = NULL;
         dict_t        *xattr = NULL;
         void          *index_gfid = NULL;
+        void          *base_indices_holder_vgfid = NULL;
         loc_t         rootloc = {0};
         struct iatt   iattr = {0};
         struct iatt   parent = {0};
@@ -809,7 +1159,7 @@ afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data,
         priv = this->private;
         if (crawl_data->crawl == FULL) {
                 afr_build_root_loc (this, dirloc);
-        } else {
+        } else if (crawl_data->crawl == INDEX) {
                 afr_build_root_loc (this, &rootloc);
                 ret = syncop_getxattr (readdir_xl, &rootloc, &xattr,
                                        GF_XATTROP_INDEX_GFID);
@@ -843,6 +1193,47 @@ afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data,
                 ret = _link_inode_update_loc (this, dirloc, &iattr);
                 if (ret)
                         goto out;
+        } else if (crawl_data->crawl == INDEX_TO_BE_HEALED) {
+                afr_build_root_loc (this, &rootloc);
+                ret = syncop_getxattr (readdir_xl, &rootloc, &xattr,
+                                       GF_BASE_INDICES_HOLDER_GFID);
+                if (ret < 0)
+                        goto out;
+                ret = dict_get_ptr (xattr, GF_BASE_INDICES_HOLDER_GFID,
+                                    &base_indices_holder_vgfid);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "index gfid empty "
+                                "on %s", readdir_xl->name);
+                        ret = -1;
+                        goto out;
+                }
+                if (!base_indices_holder_vgfid) {
+                        gf_log (this->name, GF_LOG_ERROR, "Base indices holder"
+                                "virtual gfid is null on %s", readdir_xl->name);
+                        ret = -1;
+                        goto out;
+                }
+                uuid_copy (dirloc->gfid,  base_indices_holder_vgfid);
+                dirloc->path = "";
+                dirloc->inode = inode_new (priv->root_inode->table);
+                ret = syncop_lookup (readdir_xl, dirloc, NULL, &iattr, NULL,
+                                     &parent);
+                if (ret < 0) {
+                        if (errno != ENOENT) {
+                                gf_log (this->name, GF_LOG_ERROR, "lookup "
+                                        "failed for base_indices_holder dir"
+                                        " on %s - (%s)", readdir_xl->name,
+                                        strerror (errno));
+
+                        } else {
+                                gf_log (this->name, GF_LOG_ERROR, "base_indices"
+                                        "_holder is not yet created.");
+                        }
+                        goto out;
+                }
+                ret = _link_inode_update_loc (this, dirloc, &iattr);
+                if (ret)
+                        goto out;
         }
         ret = 0;
 out:
@@ -907,6 +1298,16 @@ afr_crawl_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent,
         priv = this->private;
         if (crawl_data->crawl == FULL) {
                 ret = afr_build_child_loc (this, child, parent, entry->d_name);
+        } else if (crawl_data->crawl == INDEX_TO_BE_HEALED) {
+                ret = _build_index_loc (this, child, entry->d_name, parent);
+                if (ret)
+                        goto out;
+                child->inode = inode_new (priv->root_inode->table);
+                if (!child->inode) {
+                        ret = -1;
+                        goto out;
+                }
+                child->path = NULL;
         } else {
                 child->inode = inode_new (priv->root_inode->table);
                 if (!child->inode)
@@ -956,10 +1357,14 @@ _process_entries (xlator_t *this, loc_t *parentloc, gf_dirent_t *entries,
                 ret = crawl_data->process_entry (this, crawl_data, entry,
                                                  &entry_loc, parentloc, &iattr);
 
-                if (ret)
+                if (crawl_data->crawl == INDEX_TO_BE_HEALED && ret) {
+                       goto out;
+                } else if (ret) {
                         continue;
+                }
 
-                if (crawl_data->crawl == INDEX)
+                if ((crawl_data->crawl == INDEX) ||
+                    (crawl_data->crawl == INDEX_TO_BE_HEALED))
                         continue;
 
                 if (!IA_ISDIR (iattr.ia_type))
@@ -974,6 +1379,10 @@ _process_entries (xlator_t *this, loc_t *parentloc, gf_dirent_t *entries,
         }
         ret = 0;
 out:
+        if ((crawl_data->crawl == INDEX_TO_BE_HEALED)  && ret) {
+                gf_log (this->name, GF_LOG_ERROR,"Failed to get the hardlink "
+                        "count");
+        }
         loc_wipe (&entry_loc);
         return ret;
 }
@@ -1021,6 +1430,9 @@ _crawl_directory (fd_t *fd, loc_t *loc, afr_crawl_data_t *crawl_data)
 
                 ret = _process_entries (this, loc, &entries, &offset,
                                         crawl_data);
+                if ((ret < 0) && (crawl_data->crawl == INDEX_TO_BE_HEALED)) {
+                        goto out;
+                }
                 gf_dirent_free (&entries);
                 free_entries = _gf_false;
         }
@@ -1126,8 +1538,13 @@ afr_dir_crawl (void *data)
                 goto out;
 
         ret = afr_crawl_opendir (this, crawl_data, &fd, &dirloc);
-        if (ret)
+        if (ret) {
+                if (crawl_data->crawl == INDEX_TO_BE_HEALED) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to open base_"
+                                "indices_holder");
+                }
                 goto out;
+        }
 
         ret = _crawl_directory (fd, &dirloc, crawl_data);
         if (ret)
@@ -1141,12 +1558,102 @@ afr_dir_crawl (void *data)
 out:
         if (fd)
                 fd_unref (fd);
-        if (crawl_data->crawl == INDEX)
+        if ((crawl_data->crawl == INDEX) ||
+            (crawl_data->crawl == INDEX_TO_BE_HEALED ))
                 dirloc.path = NULL;
         loc_wipe (&dirloc);
         return ret;
 }
 
+char *
+get_crawl_type_in_string (afr_crawl_type_t crawl)
+{
+        char    *index = "INDEX";
+        char    *full  = "FULL";
+        char    *crawl_type = NULL;
+
+        if (crawl == INDEX){
+                crawl_type = index;
+        } else if (crawl == FULL) {
+                crawl_type = full;
+        }
+
+        return  crawl_type;
+}
+
+static int
+afr_allocate_crawl_event (xlator_t *this, int child, afr_crawl_type_t crawl)
+{
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     ret = 0;
+        shd_crawl_event_t       *crawl_event = NULL;
+        time_t                  get_time = 0;
+
+        priv = this->private;
+        shd = &priv->shd;
+
+        crawl_event = GF_CALLOC (sizeof (shd_crawl_event_t), 1,
+                                 gf_afr_mt_shd_crawl_event_t);
+        if (!crawl_event) {
+                ret = -1;
+                goto out;
+        }
+
+        get_time =  time(NULL);
+        if (get_time == ((time_t)-1)) {
+                 ret = -1;
+                goto out;
+        }
+
+        crawl_event->start_time_str = gf_strdup (ctime(&get_time));
+
+        crawl_event->crawl_type = get_crawl_type_in_string (crawl);
+        if (!crawl_event->crawl_type) {
+                ret = -1;
+                goto out;
+        }
+        LOCK (&priv->lock);
+        {
+                shd->crawl_events[child] = crawl_event;
+        }
+        UNLOCK (&priv->lock);
+        ret = 0;
+out:
+        return ret;
+
+}
+
+static int
+afr_put_crawl_event_in_eh (xlator_t *this, int child)
+{
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     ret = 0;
+        time_t                  get_time = 0;
+        shd_crawl_event_t       **crawl_event = NULL;
+
+        priv = this->private;
+        shd = &priv->shd;
+
+        get_time = time(NULL);
+        if (get_time == ((time_t)-1)) {
+                ret = -1;
+                goto out;
+        }
+        crawl_event = (shd_crawl_event_t**)shd->crawl_events;
+        LOCK (&priv->lock);
+        {
+                crawl_event[child]->end_time_str = gf_strdup (ctime(&get_time));
+                ret = eh_save_history (shd->statistics[child],
+                                       crawl_event[child]);
+                crawl_event[child] = NULL;
+        }
+        UNLOCK (&priv->lock);
+out:
+        return ret;
+}
+
 static int
 afr_dir_exclusive_crawl (void *data)
 {
@@ -1182,7 +1689,15 @@ afr_dir_exclusive_crawl (void *data)
         }
 
         do {
+                ret = afr_allocate_crawl_event (this, child, crawl_data->crawl);
+                if (ret)
+                        goto out;
                 afr_dir_crawl (data);
+
+                ret = afr_put_crawl_event_in_eh (this, child);
+                if (ret < 0)
+                        goto out;
+
                 LOCK (&priv->lock);
                 {
                         if (shd->pending[child] != NONE) {
@@ -1270,4 +1785,3 @@ afr_set_root_gfid (dict_t *dict)
 
         return ret;
 }
-
diff --git a/xlators/cluster/afr/src/afr-self-heald.h b/xlators/cluster/afr/src/afr-self-heald.h
index 32a8aaca5..e0c083754 100644
--- a/xlators/cluster/afr/src/afr-self-heald.h
+++ b/xlators/cluster/afr/src/afr-self-heald.h
@@ -29,6 +29,19 @@ typedef struct afr_crawl_data_ {
                               struct iatt *iattr);
 } afr_crawl_data_t;
 
+typedef struct crawl_event_stats_ {
+        uint64_t healed_count;
+        uint64_t split_brain_count;
+        uint64_t heal_failed_count;
+        char     *start_time_str;
+        char     *end_time_str;
+        char     *crawl_type;
+        gf_boolean_t crawl_inprogress;
+} shd_crawl_event_t;
+
+void _destroy_crawl_event_data (void *data);
+void _destroy_shd_event_data (void *data);
+
 typedef int (*process_entry_cbk_t) (xlator_t *this, afr_crawl_data_t *crawl_data,
                               gf_dirent_t *entry, loc_t *child, loc_t *parent,
                               struct iatt *iattr);
diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c
index 0b91f466e..20306e469 100644
--- a/xlators/cluster/afr/src/afr-transaction.c
+++ b/xlators/cluster/afr/src/afr-transaction.c
@@ -1699,7 +1699,7 @@ afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
 {
 	afr_fd_ctx_t      *fd_ctx = NULL;
 	call_frame_t      *prev_frame = NULL;
-	struct timeval     delta = {0, };
+	struct timespec    delta = {0, };
 	afr_private_t     *priv = NULL;
 	afr_local_t       *local = NULL;
 
@@ -1710,7 +1710,7 @@ afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
                 goto out;
 
 	delta.tv_sec = priv->post_op_delay_secs;
-	delta.tv_usec = 0;
+	delta.tv_nsec = 0;
 
 	pthread_mutex_lock (&fd_ctx->delay_lock);
 	{
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index 4338b799a..c724eb2ae 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -388,7 +388,8 @@ init (xlator_t *this)
                 i++;
         }
 
-        ret = gf_asprintf (&priv->sh_domain, "%s-self-heal", this->name);
+        ret = gf_asprintf (&priv->sh_domain, AFR_SH_DATA_DOMAIN_FMT,
+                           this->name);
         if (-1 == ret) {
                 ret = -ENOMEM;
                 goto out;
@@ -438,15 +439,18 @@ init (xlator_t *this)
         if (!priv->shd.timer)
                 goto out;
 
-        priv->shd.healed = eh_new (AFR_EH_HEALED_LIMIT, _gf_false);
+        priv->shd.healed = eh_new (AFR_EH_HEALED_LIMIT, _gf_false,
+                                   _destroy_shd_event_data);
         if (!priv->shd.healed)
                 goto out;
 
-        priv->shd.heal_failed = eh_new (AFR_EH_HEAL_FAIL_LIMIT, _gf_false);
+        priv->shd.heal_failed = eh_new (AFR_EH_HEAL_FAIL_LIMIT, _gf_false,
+                                        _destroy_shd_event_data);
         if (!priv->shd.heal_failed)
                 goto out;
 
-        priv->shd.split_brain = eh_new (AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false);
+        priv->shd.split_brain = eh_new (AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
+                                        _destroy_shd_event_data);
         if (!priv->shd.split_brain)
                 goto out;
 
@@ -456,7 +460,9 @@ init (xlator_t *this)
         priv->root_inode = inode_ref (this->itable->root);
         GF_OPTION_INIT ("node-uuid", priv->shd.node_uuid, str, out);
         GF_OPTION_INIT ("heal-timeout", priv->shd.timeout, int32, out);
-
+        ret = afr_initialise_statistics (this);
+        if (ret)
+                goto out;
         ret = 0;
 out:
         return ret;
@@ -493,6 +499,7 @@ struct xlator_fops fops = {
         .fentrylk    = afr_fentrylk,
 	.fallocate   = afr_fallocate,
 	.discard     = afr_discard,
+        .zerofill    = afr_zerofill,
 
         /* inode read */
         .access      = afr_access,
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 70c5a96a7..21064db58 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -28,6 +28,7 @@
 #define AFR_XATTR_PREFIX "trusted.afr"
 #define AFR_PATHINFO_HEADER "REPLICATE:"
 #define AFR_SH_READDIR_SIZE_KEY "self-heal-readdir-size"
+#define AFR_SH_DATA_DOMAIN_FMT "%s:self-heal"
 
 #define AFR_LOCKEE_COUNT_MAX    3
 #define AFR_DOM_COUNT_MAX    3
@@ -93,21 +94,24 @@ typedef struct afr_inode_ctx_ {
 typedef enum {
         NONE,
         INDEX,
+        INDEX_TO_BE_HEALED,
         FULL,
 } afr_crawl_type_t;
 
 typedef struct afr_self_heald_ {
-        gf_boolean_t     enabled;
-        gf_boolean_t     iamshd;
-        afr_crawl_type_t *pending;
-        gf_boolean_t     *inprogress;
-        afr_child_pos_t  *pos;
-        gf_timer_t       **timer;
-        eh_t             *healed;
-        eh_t             *heal_failed;
-        eh_t             *split_brain;
-        char             *node_uuid;
-        int              timeout;
+        gf_boolean_t            enabled;
+        gf_boolean_t            iamshd;
+        afr_crawl_type_t        *pending;
+        gf_boolean_t            *inprogress;
+        afr_child_pos_t         *pos;
+        gf_timer_t              **timer;
+        eh_t                    *healed;
+        eh_t                    *heal_failed;
+        eh_t                    *split_brain;
+        eh_t                    **statistics;
+        void                    **crawl_events;
+        char                    *node_uuid;
+        int                     timeout;
 } afr_self_heald_t;
 
 typedef struct _afr_private {
@@ -493,20 +497,23 @@ typedef struct _afr_local {
         int      optimistic_change_log;
 	gf_boolean_t      delayed_post_op;
 
+
 	/* Is the current writev() going to perform a stable write?
 	   i.e, is fd->flags or @flags writev param have O_SYNC or
 	   O_DSYNC?
 	*/
-	gf_boolean_t      stable_write;
+        gf_boolean_t      stable_write;
 
-	/* This write appended to the file. Nnot necessarily O_APPEND,
-	   just means the offset of write was at the end of file.
-	*/
-	gf_boolean_t      append_write;
+        /* This write appended to the file. Nnot necessarily O_APPEND,
+           just means the offset of write was at the end of file.
+        */
+        gf_boolean_t      append_write;
 
-        /*
-          This struct contains the arguments for the "continuation"
-          (scheme-like) of fops
+        int allow_sh_for_running_transaction;
+
+
+        /* This struct contains the arguments for the "continuation"
+           (scheme-like) of fops
         */
 
         int   op;
@@ -602,7 +609,9 @@ typedef struct _afr_local {
                 struct {
                         struct iatt prebuf;
                         struct iatt postbuf;
+                } inode_wfop; //common structure for all inode-write-fops
 
+                struct {
                         int32_t op_ret;
 
                         struct iovec *vector;
@@ -613,34 +622,21 @@ typedef struct _afr_local {
                 } writev;
 
                 struct {
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-                } fsync;
-
-                struct {
                         off_t offset;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
                 } truncate;
 
                 struct {
                         off_t offset;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
                 } ftruncate;
 
                 struct {
                         struct iatt in_buf;
                         int32_t valid;
-                        struct iatt preop_buf;
-                        struct iatt postop_buf;
                 } setattr;
 
                 struct {
                         struct iatt in_buf;
                         int32_t valid;
-                        struct iatt preop_buf;
-                        struct iatt postop_buf;
                 } fsetattr;
 
                 struct {
@@ -707,17 +703,21 @@ typedef struct _afr_local {
 			int32_t mode;
 			off_t offset;
 			size_t len;
-			struct iatt prebuf;
-			struct iatt postbuf;
 		} fallocate;
 
 		struct {
 			off_t offset;
 			size_t len;
-			struct iatt prebuf;
-			struct iatt postbuf;
 		} discard;
 
+                struct {
+                        off_t offset;
+                        size_t len;
+                        struct iatt prebuf;
+                        struct iatt postbuf;
+                } zerofill;
+
+
         } cont;
 
         struct {
@@ -1170,6 +1170,27 @@ afr_xattr_array_destroy (dict_t **xattr, unsigned int child_count);
 } while (0);
 
 
+#define AFR_SBRAIN_MSG "Failed on %s as split-brain is seen. Returning EIO."
+
+#define AFR_SBRAIN_CHECK_FD(fd, label) do {                              \
+        if (fd->inode && afr_is_split_brain (this, fd->inode)) {        \
+                op_errno = EIO;                                         \
+                gf_log (this->name, GF_LOG_WARNING,                     \
+                        AFR_SBRAIN_MSG ,uuid_utoa (fd->inode->gfid));   \
+                goto label;                                             \
+        }                                                               \
+} while (0)
+
+#define AFR_SBRAIN_CHECK_LOC(loc, label) do {                           \
+        if (loc->inode && afr_is_split_brain (this, loc->inode)) {      \
+                op_errno = EIO;                                         \
+                loc_path (loc, NULL);                                   \
+                gf_log (this->name, GF_LOG_WARNING,                     \
+                        AFR_SBRAIN_MSG , loc->path);                    \
+                goto label;                                             \
+        }                                                               \
+} while (0)
+
 int
 afr_fd_report_unstable_write (xlator_t *this, fd_t *fd);
 
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index b1fee655b..8f61339e6 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -22,6 +22,7 @@
 #include "dht-common.h"
 #include "defaults.h"
 #include "byte-order.h"
+#include "glusterfs-acl.h"
 
 #include <sys/time.h>
 #include <libgen.h>
@@ -154,7 +155,6 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
         int              ret = -1;
         dht_layout_t    *layout = NULL;
         dht_conf_t      *conf = NULL;
-        uint32_t         missing = 0;
 
         local = discover_frame->local;
         layout = local->layout;
@@ -191,33 +191,20 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
                         goto out;
                 }
         } else {
-                ret = dht_layout_normalize (this, &local->loc, layout,
-                                            &missing);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "normalizing failed on %s (internal error)",
-                                local->loc.path);
-                        op_errno = EIO;
-                        goto out;
-                }
-                if (missing == conf->subvolume_cnt) {
+                ret = dht_layout_normalize (this, &local->loc, layout);
+                if ((ret < 0) || ((ret > 0) && (local->op_ret != 0))) {
+                        /* either the layout is incorrect or the directory is
+                         * not found even in one subvolume.
+                         */
                         gf_log (this->name, GF_LOG_DEBUG,
-                                "normalizing failed on %s, ENOENT errors: %u)",
-                                local->loc.path, missing);
-                        op_errno = ENOENT;
-                        goto out;
-                }
-                if (ret != 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
                                 "normalizing failed on %s "
-                                "(overlaps/holes present)", local->loc.path);
-                        /* We may need to do the lookup again */
-                        /* in discover call, parent is not know, and basename
-                         * of entry is also not available. Without which we
-                         * cannot build a layout correctly to heal it. Hence
-                         * returning ESTALE */
-                        op_errno = ESTALE;
-                        goto out;
+                                "(overlaps/holes present: %s, "
+                                "ENOENT errors: %d)", local->loc.path,
+                                (ret < 0) ? "yes" : "no", (ret > 0) ? ret : 0);
+                        if ((ret > 0) && (ret == conf->subvolume_cnt)) {
+                                op_errno = ESTALE;
+                                goto out;
+                        }
                 }
 
                 if (local->inode)
@@ -421,7 +408,6 @@ dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         dht_layout_t *layout                  = NULL;
         int           ret                     = -1;
         int           is_dir                  = 0;
-        uint32_t      missing                 = 0;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, out);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -455,7 +441,7 @@ dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                         op_ret, op_errno, xattr);
 
                 if (op_ret == -1) {
-                        local->op_errno = ENOENT;
+                        local->op_errno = op_errno;
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "lookup of %s on %s returned error (%s)",
                                 local->loc.path, prev->this->name,
@@ -502,16 +488,9 @@ unlock:
                 }
 
                 if (local->op_ret == 0) {
-                        ret = dht_layout_normalize (this, &local->loc, layout,
-                                                    &missing);
-
-                        /*
-                         * Arguably, we shouldn't do self-heal just because
-                         * bricks are missing as long as there are no other
-                         * anomalies.  For now, though, just preserve the
-                         * existing behavior.
-                         */
-                        if ((ret != 0) || (missing != 0)) {
+                        ret = dht_layout_normalize (this, &local->loc, layout);
+
+                        if (ret != 0) {
                                 gf_log (this->name, GF_LOG_DEBUG,
                                         "fixing assignment on %s",
                                         local->loc.path);
@@ -5194,8 +5173,8 @@ unlock:
                  * not need to handle CHILD_DOWN event here.
                  */
                 if (conf->defrag) {
-                        ret = pthread_create (&conf->defrag->th, NULL,
-                                              gf_defrag_start, this);
+                        ret = gf_thread_create (&conf->defrag->th, NULL,
+						gf_defrag_start, this);
                         if (ret) {
                                 conf->defrag = NULL;
                                 GF_FREE (conf->defrag);
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 6ba18339b..5ccd66799 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -212,6 +212,10 @@ enum gf_defrag_status_t {
         GF_DEFRAG_STATUS_STOPPED,
         GF_DEFRAG_STATUS_COMPLETE,
         GF_DEFRAG_STATUS_FAILED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED,
 };
 typedef enum gf_defrag_status_t gf_defrag_status_t;
 
@@ -396,26 +400,18 @@ typedef enum {
         } while (0)
 
 #define is_greater_time(a, an, b, bn) (((a) < (b)) || (((a) == (b)) && ((an) < (bn))))
-
-dht_layout_t    *dht_layout_new         (xlator_t *this, int cnt);
-dht_layout_t    *dht_layout_get         (xlator_t *this, inode_t *inode);
-dht_layout_t    *dht_layout_for_subvol  (xlator_t *this, xlator_t *subvol);
-xlator_t        *dht_layout_search      (xlator_t *this, dht_layout_t *layout,
-                                         const char *name);
-int             dht_layout_normalize    (xlator_t *this, loc_t *loc,
-                                         dht_layout_t *layout,
-                                         uint32_t *missing_p);
-int             dht_layout_anomalies    (xlator_t *this, loc_t *loc,
-                                         dht_layout_t *layout,
-                                         uint32_t *holes_p,
-                                         uint32_t *overlaps_p,
-                                         uint32_t *missing_p,
-                                         uint32_t *down_p,
-                                         uint32_t *misc_p,
-                                         uint32_t *no_space_p);
-int             dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout,
-                                         xlator_t *subvol, loc_t *loc,
-                                         dict_t *xattr);
+dht_layout_t                            *dht_layout_new (xlator_t *this, int cnt);
+dht_layout_t                            *dht_layout_get (xlator_t *this, inode_t *inode);
+dht_layout_t                            *dht_layout_for_subvol (xlator_t *this, xlator_t *subvol);
+xlator_t *dht_layout_search (xlator_t   *this, dht_layout_t *layout,
+                             const char *name);
+int                                      dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout);
+int dht_layout_anomalies (xlator_t      *this, loc_t *loc, dht_layout_t *layout,
+                          uint32_t      *holes_p, uint32_t *overlaps_p,
+                          uint32_t      *missing_p, uint32_t *down_p,
+                          uint32_t      *misc_p, uint32_t *no_space_p);
+int dht_layout_dir_mismatch (xlator_t   *this, dht_layout_t *layout,
+                             xlator_t   *subvol, loc_t *loc, dict_t *xattr);
 
 xlator_t *dht_linkfile_subvol (xlator_t *this, inode_t *inode,
                                struct iatt *buf, dict_t *xattr);
@@ -699,6 +695,8 @@ int32_t dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
 		      int32_t mode, off_t offset, size_t len, dict_t *xdata);
 int32_t dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd,
 		    off_t offset, size_t len, dict_t *xdata);
+int32_t dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    off_t offset, size_t len, dict_t *xdata);
 
 int32_t dht_init (xlator_t *this);
 void    dht_fini (xlator_t *this);
@@ -783,4 +781,7 @@ dht_priv_dump (xlator_t *this);
 int32_t
 dht_inodectx_dump (xlator_t *this, inode_t *inode);
 
+int
+dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol);
+
 #endif/* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c
index 1cf343d24..311a48112 100644
--- a/xlators/cluster/dht/src/dht-helper.c
+++ b/xlators/cluster/dht/src/dht-helper.c
@@ -18,6 +18,28 @@
 #include "xlator.h"
 #include "dht-common.h"
 
+static inline int
+dht_inode_ctx_set1 (xlator_t *this, inode_t *inode, xlator_t *subvol)
+{
+        uint64_t tmp_subvol = 0;
+
+        tmp_subvol = (long)subvol;
+        return inode_ctx_set1 (inode, this, &tmp_subvol);
+}
+
+int
+dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol)
+{
+        int ret = -1;
+        uint64_t tmp_subvol = 0;
+
+        ret =  inode_ctx_get1 (inode, this, &tmp_subvol);
+        if (tmp_subvol && subvol)
+                *subvol = (xlator_t *)tmp_subvol;
+
+        return ret;
+}
+
 
 int
 dht_frame_return (call_frame_t *frame)
@@ -340,20 +362,6 @@ out:
         return local;
 }
 
-
-char *
-basestr (const char *str)
-{
-        char *basestr = NULL;
-
-        basestr = strrchr (str, '/');
-        if (basestr)
-                basestr ++;
-
-        return basestr;
-}
-
-
 xlator_t *
 dht_first_up_subvol (xlator_t *this)
 {
@@ -727,6 +735,10 @@ dht_migration_complete_check_task (void *data)
         loc_t         tmp_loc  = {0,};
         char         *path     = NULL;
         dht_conf_t   *conf     = NULL;
+        inode_t      *inode    = NULL;
+        fd_t         *iter_fd  = NULL;
+        uint64_t      tmp_subvol = 0;
+        int           open_failed = 0;
 
         this  = THIS;
         frame = data;
@@ -738,6 +750,8 @@ dht_migration_complete_check_task (void *data)
         if (!local->loc.inode && !local->fd)
                 goto out;
 
+        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
         /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
          * as root:root. If a fd is already open, access check wont be done*/
 
@@ -800,10 +814,7 @@ dht_migration_complete_check_task (void *data)
         /* update inode ctx (the layout) */
         dht_layout_unref (this, local->layout);
 
-        if (!local->loc.inode)
-                ret = dht_layout_preset (this, dst_node, local->fd->inode);
-        else
-                ret = dht_layout_preset (this, dst_node, local->loc.inode);
+        ret = dht_layout_preset (this, dst_node, inode);
         if (ret != 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "%s: could not set preset layout for subvol %s",
@@ -821,10 +832,7 @@ dht_migration_complete_check_task (void *data)
                 goto out;
         }
 
-        if (!local->loc.inode)
-                ret = dht_layout_set (this, local->fd->inode, layout);
-        else
-                ret = dht_layout_set (this, local->loc.inode, layout);
+        ret = dht_layout_set (this, inode, layout);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: failed to set the new layout",
@@ -835,41 +843,46 @@ dht_migration_complete_check_task (void *data)
         local->cached_subvol = dst_node;
         ret = 0;
 
-        if (!local->fd)
+        /* once we detect the migration complete, the inode-ctx2 is no more
+           required.. delete the ctx and also, it means, open() already
+           done on all the fd of inode */
+        ret = inode_ctx_reset1 (inode, this, &tmp_subvol);
+        if (tmp_subvol)
                 goto out;
-        /* once we detect the migration complete, the fd-ctx is no more
-           required.. delete the ctx */
-        ret = fd_ctx_del (local->fd, this, NULL);
-        if (!ret)
+
+        if (list_empty (&inode->fd_list))
                 goto out;
 
         /* perform open as root:root. There is window between linkfile
          * creation(root:root) and setattr with the correct uid/gid
          */
         SYNCTASK_SETID(0, 0);
-        /* if 'local->fd' (ie, fd based operation), send a 'open()' on
-           destination if not already done */
-        if (local->loc.inode) {
-                ret = syncop_open (dst_node, &local->loc,
-                                   local->fd->flags, local->fd);
-        } else {
-                tmp_loc.inode = local->fd->inode;
-                inode_path (local->fd->inode, NULL, &path);
-                if (path)
-                        tmp_loc.path = path;
-                ret = syncop_open (dst_node, &tmp_loc,
-                                   local->fd->flags, local->fd);
-                GF_FREE (path);
 
+        /* perform 'open()' on all the fd's present on the inode */
+        tmp_loc.inode = inode;
+        inode_path (inode, NULL, &path);
+        if (path)
+                tmp_loc.path = path;
+        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (fd_is_anonymous (iter_fd))
+                        continue;
+
+                ret = syncop_open (dst_node, &tmp_loc,
+                                   iter_fd->flags, iter_fd);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to open "
+                                "the fd (%p, flags=0%o) on file %s @ %s",
+                                iter_fd, iter_fd->flags, path, dst_node->name);
+                        open_failed = 1;
+                }
         }
+        GF_FREE (path);
+
         SYNCTASK_SETID (frame->root->uid, frame->root->gid);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to send open() on target file at %s",
-                        local->loc.path, dst_node->name);
+        if (open_failed) {
+                ret = -1;
                 goto out;
         }
-
         ret = 0;
 out:
 
@@ -914,6 +927,9 @@ dht_rebalance_inprogress_task (void *data)
         struct iatt   stbuf    = {0,};
         loc_t         tmp_loc  = {0,};
         dht_conf_t   *conf     = NULL;
+        inode_t      *inode    = NULL;
+        fd_t         *iter_fd  = NULL;
+        int           open_failed = 0;
 
         this  = THIS;
         frame = data;
@@ -925,6 +941,8 @@ dht_rebalance_inprogress_task (void *data)
         if (!local->loc.inode && !local->fd)
                 goto out;
 
+        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
         /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
          * as root:root. If a fd is already open, access check wont be done*/
         if (local->loc.inode) {
@@ -976,35 +994,47 @@ dht_rebalance_inprogress_task (void *data)
         }
 
         ret = 0;
+
+        if (list_empty (&inode->fd_list))
+                goto done;
+
         /* perform open as root:root. There is window between linkfile
          * creation(root:root) and setattr with the correct uid/gid
          */
         SYNCTASK_SETID (0, 0);
-        if (local->loc.inode) {
-                ret = syncop_open (dst_node, &local->loc,
-                                   local->fd->flags, local->fd);
-        } else {
-                tmp_loc.inode = local->fd->inode;
-                inode_path (local->fd->inode, NULL, &path);
-                if (path)
-                        tmp_loc.path = path;
+
+        tmp_loc.inode = inode;
+        inode_path (inode, NULL, &path);
+        if (path)
+                tmp_loc.path = path;
+
+        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (fd_is_anonymous (iter_fd))
+                        continue;
+
                 ret = syncop_open (dst_node, &tmp_loc,
-                                   local->fd->flags, local->fd);
-                GF_FREE (path);
+                                   iter_fd->flags, iter_fd);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to send open "
+                                "the fd (%p, flags=0%o) on file %s @ %s",
+                                iter_fd, iter_fd->flags, path, dst_node->name);
+                        open_failed = 1;
+                }
         }
+        GF_FREE (path);
 
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to send open() on target file at %s",
-                        local->loc.path, dst_node->name);
+        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+
+        if (open_failed) {
+                ret = -1;
                 goto out;
         }
 
-        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
-        ret = fd_ctx_set (local->fd, this, (uint64_t)(long)dst_node);
+done:
+        ret = dht_inode_ctx_set1 (this, inode, dst_node);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set fd-ctx target file at %s",
+                        "%s: failed to set inode-ctx target file at %s",
                         local->loc.path, dst_node->name);
                 goto out;
         }
diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c
index 10f59ab26..ece84151a 100644
--- a/xlators/cluster/dht/src/dht-inode-read.c
+++ b/xlators/cluster/dht/src/dht-inode-read.c
@@ -130,10 +130,11 @@ int
 dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
 {
-        uint64_t      tmp_subvol = 0;
+        xlator_t     *subvol = 0;
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        inode_t      *inode = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -157,19 +158,20 @@ dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local->op_errno = op_errno;
         /* Check if the rebalance phase2 is true */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                if (local->fd)
-                        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-                if (ret) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                ret = dht_inode_ctx_get1 (this, inode, &subvol);
+                if (!subvol) {
                         /* Phase 2 of migration */
                         local->rebalance.target_op_fn = dht_attr2;
                         ret = dht_rebalance_complete_check (this, frame);
+                        if (!ret)
+                                return 0;
                 } else {
                         /* value is already set in fd_ctx, that means no need
                            to check for whether its complete or not. */
                         dht_attr2 (this, frame, 0);
-                }
-                if (!ret)
                         return 0;
+                }
         }
 
 out:
@@ -382,6 +384,8 @@ dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 {
         dht_local_t *local      = NULL;
         int          ret        = 0;
+        inode_t     *inode      = NULL;
+        xlator_t    *subvol = 0;
 
         local = frame->local;
         if (!local) {
@@ -400,17 +404,18 @@ dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local->op_errno = op_errno;
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
                 /* File would be migrated to other node */
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (ret) {
+                ret = dht_inode_ctx_get1 (this, inode, &subvol);
+                if (!subvol) {
                         local->rebalance.target_op_fn = dht_readv2;
                         ret = dht_rebalance_complete_check (this, frame);
+                        if (!ret)
+                                return 0;
                 } else {
                         /* value is already set in fd_ctx, that means no need
                            to check for whether its complete or not. */
                         dht_readv2 (this, frame, 0);
-                }
-                if (!ret)
                         return 0;
+                }
         }
 
 out:
@@ -616,8 +621,9 @@ int
 dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           ret = -1;
+        dht_local_t  *local  = NULL;
+        inode_t      *inode  = NULL;
+        xlator_t     *subvol = 0;
 
         local = frame->local;
 
@@ -627,8 +633,8 @@ dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
 
         /* If context is set, then send flush() it to the destination */
-        ret = fd_ctx_get (local->fd, this, NULL);
-        if (!ret) {
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (subvol) {
                 dht_flush2 (this, frame, 0);
                 return 0;
         }
@@ -644,14 +650,10 @@ dht_flush2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
@@ -713,6 +715,8 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        inode_t      *inode = NULL;
+        xlator_t     *subvol = 0;
 
         local = frame->local;
         prev = cookie;
@@ -734,8 +738,8 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
         }
 
         local->op_errno = op_errno;
-        ret = fd_ctx_get (local->fd, this, NULL);
-        if (ret) {
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (!subvol) {
                 local->rebalance.target_op_fn = dht_fsync2;
 
                 /* Check if the rebalance phase1 is true */
@@ -750,11 +754,12 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                         ret = dht_rebalance_complete_check (this, frame);
                 }
+                if (!ret)
+                        return 0;
         } else {
                 dht_fsync2 (this, frame, 0);
-        }
-        if (!ret)
                 return 0;
+        }
 
 out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
@@ -770,15 +775,10 @@ dht_fsync2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
-
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
         if (!subvol)
                 subvol = local->cached_subvol;
 
diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c
index 9bcd84ae1..4b3f3a049 100644
--- a/xlators/cluster/dht/src/dht-inode-write.c
+++ b/xlators/cluster/dht/src/dht-inode-write.c
@@ -21,6 +21,7 @@ int dht_truncate2 (xlator_t *this, call_frame_t *frame, int ret);
 int dht_setattr2 (xlator_t *this, call_frame_t *frame, int ret);
 int dht_fallocate2(xlator_t *this, call_frame_t *frame, int op_ret);
 int dht_discard2(xlator_t *this, call_frame_t *frame, int op_ret);
+int dht_zerofill2(xlator_t *this, call_frame_t *frame, int op_ret);
 
 int
 dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
@@ -29,6 +30,7 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 {
         dht_local_t *local = NULL;
         int          ret   = -1;
+        xlator_t    *subvol = NULL;
 
         if (op_ret == -1 && (op_errno != ENOENT)) {
                 goto out;
@@ -65,8 +67,8 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
                 dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
 
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (!ret) {
+                ret = dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
                         dht_writev2 (this, frame, 0);
                         return 0;
                 }
@@ -90,14 +92,10 @@ dht_writev2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
@@ -172,6 +170,8 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        xlator_t    *subvol = NULL;
+        inode_t      *inode = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -213,8 +213,9 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
                 dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
                 dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (!ret || !local->fd) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                dht_inode_ctx_get1 (this, inode, &subvol);
+                if (subvol) {
                         dht_truncate2 (this, frame, 0);
                         return 0;
                 }
@@ -238,16 +239,13 @@ dht_truncate2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
+        inode_t      *inode = NULL;
 
         local = frame->local;
 
-        if (local->fd)
-                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        inode = local->fd ? local->fd->inode : local->loc.inode;
 
+        dht_inode_ctx_get1 (this, inode, &subvol);
         if (!subvol)
                 subvol = local->cached_subvol;
 
@@ -359,6 +357,7 @@ dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        xlator_t    *subvol = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -398,8 +397,8 @@ dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
         if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
                 dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
                 dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (!ret) {
+                dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
                         dht_fallocate2 (this, frame, 0);
                         return 0;
                 }
@@ -422,15 +421,10 @@ dht_fallocate2(xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        if (local->fd)
-                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
@@ -497,6 +491,7 @@ dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        xlator_t    *subvol = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -536,8 +531,8 @@ dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
         if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
                 dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
                 dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (!ret) {
+                dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
                         dht_discard2 (this, frame, 0);
                         return 0;
                 }
@@ -560,15 +555,10 @@ dht_discard2(xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        if (local->fd)
-                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
@@ -624,6 +614,141 @@ err:
         return 0;
 }
 
+int
+dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev  = NULL;
+        int           ret   = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && (op_errno != ENOENT)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_zerofill2;
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                ret = fd_ctx_get (local->fd, this, NULL);
+                if (!ret) {
+                        dht_zerofill2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (zerofill, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_zerofill2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local          = NULL;
+        xlator_t     *subvol         = NULL;
+        uint64_t      tmp_subvol     = 0;
+        int           ret            = -1;
+
+        local = frame->local;
+
+        if (local->fd)
+                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
+        if (!ret)
+                subvol = (xlator_t *)(long)tmp_subvol;
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        STACK_WIND(frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
+                   local->fd, local->rebalance.offset, local->rebalance.size,
+                   NULL);
+
+        return 0;
+}
+
+int
+dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
+{
+        xlator_t     *subvol       = NULL;
+        int           op_errno     = -1;
+        dht_local_t  *local        = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_ZEROFILL);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.offset = offset;
+        local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
+                    fd, offset, len, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+
 /* handle cases of migration here for 'setattr()' calls */
 int
 dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
@@ -675,15 +800,13 @@ dht_setattr2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
+        inode_t      *inode = NULL;
 
         local = frame->local;
 
-        if (local->fd)
-                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get1 (this, inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c
index 07e8cbae4..38e9970a7 100644
--- a/xlators/cluster/dht/src/dht-layout.c
+++ b/xlators/cluster/dht/src/dht-layout.c
@@ -454,12 +454,19 @@ dht_layout_entry_cmp (dht_layout_t *layout, int i, int j)
 {
         int64_t diff = 0;
 
+        /* swap zero'ed out layouts to front, if needed */
+        if (!layout->list[j].start && !layout->list[j].stop) {
+                diff = (int64_t) layout->list[i].stop
+                       - (int64_t) layout->list[j].stop;
+                       goto out;
+        }
         if (layout->list[i].err || layout->list[j].err)
                 diff = layout->list[i].err - layout->list[j].err;
         else
                 diff = (int64_t) layout->list[i].start
                         - (int64_t) layout->list[j].start;
 
+out:
         return diff;
 }
 
@@ -534,13 +541,13 @@ dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
                 case -1:
                 case ENOENT:
                         missing++;
-                        break;
+                        continue;
                 case ENOTCONN:
                         down++;
-                        break;
+                        continue;
                 case ENOSPC:
                         no_space++;
-                        break;
+                        continue;
                 case 0:
                         /* if err == 0 and start == stop, then it is a non misc++;
                          * participating subvolume(spread-cnt). Then, do not
@@ -552,6 +559,7 @@ dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
                         break;
                 default:
                         misc++;
+                        continue;
                  }
 
                 is_virgin = 0;
@@ -593,8 +601,7 @@ dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
 
 
 int
-dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout,
-                      uint32_t *missing_p)
+dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout)
 {
         int          ret   = 0;
         int          i = 0;
@@ -606,7 +613,6 @@ dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout,
 
         ret = dht_layout_sort (layout);
         if (ret == -1) {
-                /* defensive coding; this can't happen currently */
                 gf_log (this->name, GF_LOG_WARNING,
                         "sort failed?! how the ....");
                 goto out;
@@ -616,26 +622,23 @@ dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout,
                                     &holes, &overlaps,
                                     &missing, &down, &misc, NULL);
         if (ret == -1) {
-                /* defensive coding; this can't happen currently */
                 gf_log (this->name, GF_LOG_WARNING,
                         "error while finding anomalies in %s -- not good news",
                         loc->path);
                 goto out;
         }
 
-        ret = holes + overlaps;
-        if (ret) {
+        if (holes || overlaps) {
                 if (missing == layout->cnt) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "directory %s looked up first time",
                                 loc->path);
                 } else {
                         gf_log (this->name, GF_LOG_INFO,
-                                "found anomalies in %s. holes=%d overlaps=%d"
-                                " missing=%d down=%d misc=%d",
-                                loc->path, holes, overlaps, missing, down,
-                                misc);
+                                "found anomalies in %s. holes=%d overlaps=%d",
+                                loc->path, holes, overlaps);
                 }
+                ret = -1;
         }
 
         for (i = 0; i < layout->cnt; i++) {
@@ -650,14 +653,14 @@ dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout,
                                           (layout->list[i].xlator ?
                                            layout->list[i].xlator->name
                                            : "<>"));
+                        if ((layout->list[i].err == ENOENT) && (ret >= 0)) {
+                                ret++;
+                        }
                 }
         }
 
 
 out:
-        if (missing_p) {
-                *missing_p = missing;
-        }
         return ret;
 }
 
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 5300ff2fe..bcb19f23e 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -243,7 +243,7 @@ out:
 
 static inline int
 __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struct iatt *stbuf,
-                                 dict_t *dict, fd_t **dst_fd)
+                                 dict_t *dict, fd_t **dst_fd, dict_t *xattr)
 {
         xlator_t    *this = NULL;
         int          ret  = -1;
@@ -307,6 +307,12 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc
                 goto out;
         }
 
+        ret = syncop_fsetxattr (to, fd, xattr, 0);
+        if (ret == -1)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set xattr on %s (%s)",
+                        loc->path, to->name, strerror (errno));
+
         ret = syncop_ftruncate (to, fd, stbuf->ia_size);
         if (ret < 0)
                 gf_log (this->name, GF_LOG_ERROR,
@@ -340,6 +346,9 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
         int             ret        = -1;
         xlator_t       *this       = NULL;
 
+        uint64_t        src_statfs_blocks = 1;
+        uint64_t        dst_statfs_blocks = 1;
+
         this = THIS;
 
         ret = syncop_statfs (from, loc, &src_statfs);
@@ -363,22 +372,34 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
         if (flag != GF_DHT_MIGRATE_DATA)
                 goto check_avail_space;
 
-        if (((dst_statfs.f_bavail *
-              dst_statfs.f_bsize) / GF_DISK_SECTOR_SIZE) <
-            (((src_statfs.f_bavail * src_statfs.f_bsize) /
-              GF_DISK_SECTOR_SIZE) - stbuf->ia_blocks)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "data movement attempted from node (%s) with"
-                        " higher disk space to a node (%s) with "
-                        "lesser disk space (%s)", from->name,
-                        to->name, loc->path);
-
-                /* this is not a 'failure', but we don't want to
-                   consider this as 'success' too :-/ */
-                ret = 1;
-                goto out;
+        /* Check:
+           During rebalance `migrate-data` - Destination subvol experiences
+           a `reduction` in 'blocks' of free space, at the same time source
+           subvol gains certain 'blocks' of free space. A valid check is
+           necessary here to avoid errorneous move to destination where
+           the space could be scantily available.
+         */
+        if (stbuf) {
+                dst_statfs_blocks = ((dst_statfs.f_bavail *
+                                      dst_statfs.f_bsize) /
+                                     GF_DISK_SECTOR_SIZE);
+                src_statfs_blocks = ((src_statfs.f_bavail *
+                                      src_statfs.f_bsize) /
+                                     GF_DISK_SECTOR_SIZE);
+                if ((dst_statfs_blocks - stbuf->ia_blocks) <
+                    (src_statfs_blocks + stbuf->ia_blocks)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "data movement attempted from node (%s) with"
+                                " higher disk space to a node (%s) with "
+                                "lesser disk space (%s)", from->name,
+                                to->name, loc->path);
+
+                        /* this is not a 'failure', but we don't want to
+                           consider this as 'success' too :-/ */
+                        ret = 1;
+                        goto out;
+                }
         }
-
 check_avail_space:
         if (((dst_statfs.f_bavail * dst_statfs.f_bsize) /
               GF_DISK_SECTOR_SIZE) < stbuf->ia_blocks) {
@@ -708,9 +729,16 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 goto out;
         }
 
+        /* TODO: move all xattr related operations to fd based operations */
+        ret = syncop_listxattr (from, loc, &xattr);
+        if (ret == -1)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to get xattr from %s (%s)",
+                        loc->path, from->name, strerror (errno));
+
         /* create the destination, with required modes/xattr */
         ret = __dht_rebalance_create_dst_file (to, from, loc, &stbuf,
-                                               dict, &dst_fd);
+                                               dict, &dst_fd, xattr);
         if (ret)
                 goto out;
 
@@ -727,6 +755,7 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 goto out;
         }
 
+
         ret = syncop_fstat (from, src_fd, &stbuf);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "failed to lookup %s on %s (%s)",
@@ -756,19 +785,6 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 goto out;
         }
 
-        /* TODO: move all xattr related operations to fd based operations */
-        ret = syncop_listxattr (from, loc, &xattr);
-        if (ret == -1)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to get xattr from %s (%s)",
-                        loc->path, from->name, strerror (errno));
-
-        ret = syncop_setxattr (to, loc, xattr, 0);
-        if (ret == -1)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set xattr on %s (%s)",
-                        loc->path, to->name, strerror (errno));
-
         /* TODO: Sync the locks */
 
         ret = syncop_fsync (to, dst_fd, 0);
@@ -1760,6 +1776,8 @@ log:
         case GF_DEFRAG_STATUS_FAILED:
                 status = "failed";
                 break;
+        default:
+                break;
         }
 
         gf_log (THIS->name, GF_LOG_INFO, "Rebalance is %s. Time taken is %.2f "
diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c
index b220a0e25..3fe96b1c7 100644
--- a/xlators/cluster/dht/src/dht-selfheal.c
+++ b/xlators/cluster/dht/src/dht-selfheal.c
@@ -17,6 +17,7 @@
 #include "glusterfs.h"
 #include "xlator.h"
 #include "dht-common.h"
+#include "glusterfs-acl.h"
 
 #define DHT_SET_LAYOUT_RANGE(layout,i,srt,chunk,cnt,path)    do {       \
                 layout->list[i].start = srt;                            \
@@ -564,9 +565,33 @@ dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
 
         for (i = 0; i < layout->cnt; i++) {
                 err = layout->list[i].err;
-                if (err == -1 || err == 0) {
-                        layout->list[i].err = -1;
+                if (err == -1 || err == 0 || err == ENOENT) {
+			/* Setting list[i].err = -1 is an indication for
+			   dht_selfheal_layout_new_directory() to assign
+			   a range. We set it to -1 based on any one of
+			   the three criteria:
+
+			   - err == -1 already, which means directory
+			     existed but layout was not set on it.
+
+			   - err == 0, which means directory exists and
+			     has an old layout piece which will be
+			     overwritten now.
+
+			   - err == ENOENT, which means directory does
+			     not exist (possibly racing with mkdir or
+			     finishing half done mkdir). The missing
+			     directory will be attempted to be recreated.
+
+			     It is important to note that it is safe
+			     to race with mkdir() as self-heal and
+			     mkdir are idempotent operations. Both will
+			     strive to set the directory and layouts to
+			     the same final state.
+			*/
                         count++;
+			if (!err)
+				layout->list[i].err = -1;
                 }
         }
 
@@ -776,7 +801,7 @@ dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
         DHT_RESET_LAYOUT_RANGE (layout);
         for (i = start_subvol; i < layout->cnt; i++) {
                 err = layout->list[i].err;
-                if (err == -1) {
+                if (err == -1 || err == ENOENT) {
                         DHT_SET_LAYOUT_RANGE(layout, i, start, chunk,
                                              cnt, loc->path);
                         if (--cnt == 0) {
@@ -789,7 +814,7 @@ dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
 
         for (i = 0; i < start_subvol; i++) {
                 err = layout->list[i].err;
-                if (err == -1) {
+                if (err == -1 || err == ENOENT) {
                         DHT_SET_LAYOUT_RANGE(layout, i, start, chunk,
                                              cnt, loc->path);
                         if (--cnt == 0) {
diff --git a/xlators/cluster/dht/src/dht.c b/xlators/cluster/dht/src/dht.c
index 0349b63a9..fc0ca2f77 100644
--- a/xlators/cluster/dht/src/dht.c
+++ b/xlators/cluster/dht/src/dht.c
@@ -72,6 +72,7 @@ struct xlator_fops fops = {
         .fsetattr    = dht_fsetattr,
 	.fallocate   = dht_fallocate,
 	.discard     = dht_discard,
+        .zerofill    = dht_zerofill,
 };
 
 struct xlator_dumpops dumpops = {
diff --git a/xlators/cluster/stripe/src/stripe.c b/xlators/cluster/stripe/src/stripe.c
index 8faaec196..69b510e23 100644
--- a/xlators/cluster/stripe/src/stripe.c
+++ b/xlators/cluster/stripe/src/stripe.c
@@ -4107,6 +4107,191 @@ err:
 }
 
 int32_t
+stripe_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt    = 0;
+        stripe_local_t *local      = NULL;
+        stripe_local_t *mlocal     = NULL;
+        call_frame_t   *prev       = NULL;
+        call_frame_t   *mframe     = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+        mframe = local->orig_frame;
+        mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+                        mlocal->prebuf_blocks  += prebuf->ia_blocks;
+                        mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+                        correct_file_size(prebuf, mlocal->fctx, prev);
+                        correct_file_size(postbuf, mlocal->fctx, prev);
+
+                        if (mlocal->prebuf_size < prebuf->ia_size)
+                                mlocal->prebuf_size = prebuf->ia_size;
+                        if (mlocal->postbuf_size < postbuf->ia_size)
+                                mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+                /* return the first failure */
+                if (mlocal->op_ret == 0) {
+                        mlocal->op_ret = op_ret;
+                        mlocal->op_errno = op_errno;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+                mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+                mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+                mlocal->post_buf.ia_size = mlocal->postbuf_size;
+                mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (zerofill, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+        STRIPE_STACK_DESTROY(frame);
+        return 0;
+}
+
+int32_t
+stripe_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               size_t len, dict_t *xdata)
+{
+        stripe_local_t   *local            = NULL;
+        stripe_fd_ctx_t  *fctx             = NULL;
+        int32_t           op_errno         = 1;
+        int32_t           idx              = 0;
+        int32_t           offset_offset    = 0;
+        int32_t           remaining_size   = 0;
+        off_t             fill_size        = 0;
+        uint64_t          stripe_size      = 0;
+        uint64_t          tmp_fctx         = 0;
+        off_t             dest_offset      = 0;
+        call_frame_t     *fframe           = NULL;
+        stripe_local_t   *flocal           = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+        local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+                fframe = copy_frame(frame);
+                flocal = mem_get0(this->local_pool);
+                if (!flocal) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+                flocal->orig_frame = frame;
+                fframe->local = flocal;
+
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+                dest_offset = offset + offset_offset;
+                if (fctx->stripe_coalesce)
+                        dest_offset = coalesced_offset(dest_offset,
+                                                       local->stripe_size,
+                                                       fctx->stripe_count);
+
+                STACK_WIND(fframe, stripe_zerofill_cbk, fctx->xl_array[idx],
+                           fctx->xl_array[idx]->fops->zerofill, fd,
+                           dest_offset, fill_size, xdata);
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+        if (fframe)
+                STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int32_t
 stripe_release (xlator_t *this, fd_t *fd)
 {
 	return 0;
@@ -4280,6 +4465,37 @@ stripe_setxattr_cbk (call_frame_t *frame, void *cookie,
         return 0;
 }
 
+#ifdef HAVE_BD_XLATOR
+int
+stripe_is_bd (dict_t *this, char *key, data_t *value, void *data)
+{
+        gf_boolean_t *is_bd = data;
+
+        if (data == NULL)
+                return 0;
+
+        if (XATTR_IS_BD (key))
+            *is_bd = _gf_true;
+
+        return 0;
+}
+
+inline gf_boolean_t
+stripe_setxattr_is_bd (dict_t *dict)
+{
+        gf_boolean_t is_bd = _gf_false;
+
+        if (dict == NULL)
+                goto out;
+
+        dict_foreach (dict, stripe_is_bd, &is_bd);
+out:
+        return is_bd;
+}
+#else
+#define stripe_setxattr_is_bd(dict) _gf_false
+#endif
+
 int
 stripe_setxattr (call_frame_t *frame, xlator_t *this,
                  loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
@@ -4289,6 +4505,7 @@ stripe_setxattr (call_frame_t *frame, xlator_t *this,
         stripe_private_t *priv     = NULL;
         stripe_local_t   *local    = NULL;
         int               i        = 0;
+        gf_boolean_t      is_bd    = _gf_false;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -4311,11 +4528,15 @@ stripe_setxattr (call_frame_t *frame, xlator_t *this,
         local->wind_count = priv->child_count;
         local->op_ret = local->op_errno = 0;
 
+        is_bd = stripe_setxattr_is_bd (dict);
+
         /**
          * Set xattrs for directories on all subvolumes. Additionally
-         * this power is only given to a special client.
+         * this power is only given to a special client. Bd xlator
+         * also needs xattrs for regular files (ie LVs)
          */
-        if ((frame->root->pid == GF_CLIENT_PID_GSYNCD) && IA_ISDIR (loc->inode->ia_type)) {
+        if (((frame->root->pid == GF_CLIENT_PID_GSYNCD) &&
+             IA_ISDIR (loc->inode->ia_type)) || is_bd) {
                 for (i = 0; i < priv->child_count; i++, trav = trav->next) {
                         STACK_WIND (frame, stripe_setxattr_cbk,
                                     trav->xlator, trav->xlator->fops->setxattr,
@@ -4346,21 +4567,21 @@ stripe_fsetxattr_cbk (call_frame_t *frame, void *cookie,
 
 
 int
-stripe_is_lockinfo (dict_t *this,
-                    char *key,
-                    data_t *value,
-                    void *data)
+stripe_is_special_key (dict_t *this,
+                       char *key,
+                       data_t *value,
+                       void *data)
 {
-        gf_boolean_t *is_lockinfo = NULL;
+        gf_boolean_t *is_special = NULL;
 
         if (data == NULL) {
                 goto out;
         }
 
-        is_lockinfo = data;
+        is_special = data;
 
-        if (XATTR_IS_LOCKINFO (key))
-                *is_lockinfo = _gf_true;
+        if (XATTR_IS_LOCKINFO (key) || XATTR_IS_BD (key))
+                *is_special = _gf_true;
 
 out:
         return 0;
@@ -4437,7 +4658,7 @@ stripe_fsetxattr_is_special (dict_t *dict)
                 goto out;
         }
 
-        dict_foreach (dict, stripe_is_lockinfo, &is_spl);
+        dict_foreach (dict, stripe_is_special_key, &is_spl);
 
 out:
         return is_spl;
@@ -5558,6 +5779,7 @@ struct xlator_fops fops = {
         .readdirp       = stripe_readdirp,
 	.fallocate	= stripe_fallocate,
 	.discard	= stripe_discard,
+        .zerofill       = stripe_zerofill,
 };
 
 struct xlator_cbks cbks = {
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index 4f27a2e41..7fb697ae4 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -1746,6 +1746,16 @@ io_stats_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
 	return 0;
 }
 
+int
+io_stats_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS(frame, ZEROFILL);
+        STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+        return 0;
+}
 
 int
 io_stats_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
@@ -2441,6 +2451,18 @@ io_stats_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
 	return 0;
 }
 
+int
+io_stats_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 size_t len, dict_t *xdata)
+{
+        START_FOP_LATENCY(frame);
+
+        STACK_WIND(frame, io_stats_zerofill_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+
+        return 0;
+}
+
 
 int
 io_stats_lk (call_frame_t *frame, xlator_t *this,
@@ -2870,6 +2892,7 @@ struct xlator_fops fops = {
         .fsetattr    = io_stats_fsetattr,
 	.fallocate   = io_stats_fallocate,
 	.discard     = io_stats_discard,
+        .zerofill    = io_stats_zerofill,
 };
 
 struct xlator_cbks cbks = {
diff --git a/xlators/debug/trace/src/trace.c b/xlators/debug/trace/src/trace.c
index a9c11babe..c9d839356 100644
--- a/xlators/debug/trace/src/trace.c
+++ b/xlators/debug/trace/src/trace.c
@@ -3134,7 +3134,7 @@ init (xlator_t *this)
         gf_log (this->name, GF_LOG_DEBUG, "logging to history %s",
                 (conf->log_history == _gf_true)?"enabled":"disabled");
 
-        history = eh_new (history_size, _gf_false);
+        history = eh_new (history_size, _gf_false, NULL);
         if (!history) {
                 gf_log (this->name, GF_LOG_ERROR, "event history cannot be "
                         "initialized");
diff --git a/xlators/encryption/Makefile.am b/xlators/encryption/Makefile.am
index 2cbde680f..36efc6698 100644
--- a/xlators/encryption/Makefile.am
+++ b/xlators/encryption/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = rot-13
+SUBDIRS = rot-13 crypt
 
 CLEANFILES = 
diff --git a/xlators/encryption/crypt/Makefile.am b/xlators/encryption/crypt/Makefile.am
new file mode 100644
index 000000000..d471a3f92
--- /dev/null
+++ b/xlators/encryption/crypt/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES = 
diff --git a/xlators/encryption/crypt/src/Makefile.am b/xlators/encryption/crypt/src/Makefile.am
new file mode 100644
index 000000000..faadd117f
--- /dev/null
+++ b/xlators/encryption/crypt/src/Makefile.am
@@ -0,0 +1,24 @@
+if ENABLE_CRYPT_XLATOR
+
+xlator_LTLIBRARIES = crypt.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/encryption
+
+crypt_la_LDFLAGS = -module -avoidversion -lssl -lcrypto
+
+crypt_la_SOURCES = keys.c data.c metadata.c atom.c crypt.c
+crypt_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
+
+else
+
+noinst_DIST = keys.c data.c metadata.c atom.c crypt.c
+noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
+
+endif
+\ No newline at end of file
diff --git a/xlators/encryption/crypt/src/atom.c b/xlators/encryption/crypt/src/atom.c
new file mode 100644
index 000000000..1ec41495c
--- /dev/null
+++ b/xlators/encryption/crypt/src/atom.c
@@ -0,0 +1,962 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+/*
+ *                              Glossary
+ *
+ *
+ * cblock            (or cipher block). A logical unit in a file.
+ *                   cblock size is defined as the number of bits
+ *                   in an input (or output) block of the block
+ *                   cipher (*). Cipher block size is a property of
+ *                   cipher algorithm. E.g. cblock size is 64 bits
+ *                   for DES, 128 bits for AES, etc.
+ *
+ * atomic cipher     A cipher algorithm, which requires some chunks of
+ * algorithm         text to be padded at left and(or) right sides before
+ *                   cipher transaform.
+ *
+ *
+ * block (atom)      Minimal chunk of file's data, which doesn't require
+ *                   padding. We'll consider logical units in a file of
+ *                   block size (atom size).
+ *
+ * cipher algorithm  Atomic cipher algorithm, which requires the last
+ * with EOF issue    incomplete cblock in a file to be padded with some
+ *                   data (usually zeros).
+ *
+ *
+ * operation, which  reading/writing from offset, which is not aligned to
+ * forms a gap at    to atom size
+ * the beginning
+ *
+ *
+ * operation, which  reading/writing count bytes starting from offset off,
+ * forms a gap at    so that off+count is not aligned to atom_size
+ * the end
+ *
+ * head block        the first atom affected by an operation, which forms
+ *                   a gap at the beginning, or(and) at the end.
+ *                   Сomment. Head block has at least one gap (either at
+ *                   the beginning, or at the end)
+ *
+ *
+ * tail block        the last atom different from head, affected by an
+ *                   operation, which forms a gap at the end.
+ *                   Сomment: Tail block has exactly one gap (at the end).
+ *
+ *
+ * partial block     head or tail block
+ *
+ *
+ * full block        block without gaps.
+ *
+ *
+ * (*) Recommendation for Block Cipher Modes of Operation
+ *     Methods and Techniques
+ *     NIST Special Publication 800-38A Edition 2001
+ */
+
+/*
+ * atom->offset_at()
+ */
+static off_t offset_at_head(struct avec_config *conf)
+{
+	return conf->aligned_offset;
+}
+
+static off_t offset_at_hole_head(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_head(get_hole_conf(frame));
+}
+
+static off_t offset_at_data_head(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_head(get_data_conf(frame));
+}
+
+
+static off_t offset_at_tail(struct avec_config *conf,
+				    struct object_cipher_info *object)
+{
+	return conf->aligned_offset +
+		(conf->off_in_head ? get_atom_size(object) : 0) +
+		(conf->nr_full_blocks << get_atom_bits(object));
+}
+
+static off_t offset_at_hole_tail(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_tail(get_hole_conf(frame), object);
+}
+
+
+static off_t offset_at_data_tail(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_tail(get_data_conf(frame), object);
+}
+
+static off_t offset_at_full(struct avec_config *conf,
+				    struct object_cipher_info *object)
+{
+	return conf->aligned_offset +
+		(conf->off_in_head ? get_atom_size(object) : 0);
+}
+
+static off_t offset_at_data_full(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_full(get_data_conf(frame), object);
+}
+
+static off_t offset_at_hole_full(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_full(get_hole_conf(frame), object);
+}
+
+/*
+ * atom->io_size_nopad()
+ */
+
+static uint32_t io_size_nopad_head(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{
+	uint32_t gap_at_beg;
+	uint32_t gap_at_end;
+
+	check_head_block(conf);
+
+	gap_at_beg = conf->off_in_head;
+
+	if (has_tail_block(conf) || has_full_blocks(conf) || conf->off_in_tail == 0 )
+		gap_at_end = 0;
+	else
+		gap_at_end = get_atom_size(object) - conf->off_in_tail;
+
+	return get_atom_size(object) - (gap_at_beg + gap_at_end);
+}
+
+static uint32_t io_size_nopad_tail(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{
+	check_tail_block(conf);
+	return conf->off_in_tail;
+}
+
+static uint32_t io_size_nopad_full(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{	
+	check_full_block(conf);
+	return get_atom_size(object);
+}
+
+static uint32_t io_size_nopad_data_head(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_head(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_head(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_head(get_hole_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_data_tail(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_tail(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_tail(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_tail(get_hole_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_data_full(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_full(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_full(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_full(get_hole_conf(frame), object);
+}
+
+static uint32_t offset_in_head(struct avec_config *conf)
+{
+	check_cursor_head(conf);
+
+	return conf->off_in_head;
+}
+
+static uint32_t offset_in_tail(call_frame_t *frame,
+			       struct object_cipher_info *object)
+{
+	return 0;
+}
+
+static uint32_t offset_in_full(struct avec_config *conf,
+			       struct object_cipher_info *object)
+{
+	check_cursor_full(conf);
+
+	if (has_head_block(conf))
+		return (conf->cursor - 1) << get_atom_bits(object);
+	else
+		return conf->cursor << get_atom_bits(object);
+}
+
+static uint32_t offset_in_data_head(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_head(get_data_conf(frame));
+}
+
+static uint32_t offset_in_hole_head(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_head(get_hole_conf(frame));
+}
+
+static uint32_t offset_in_data_full(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_full(get_data_conf(frame), object);
+}
+
+static uint32_t offset_in_hole_full(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_full(get_hole_conf(frame), object);
+}
+
+/*
+ * atom->rmw()
+ */
+/*
+ * Pre-conditions:
+ * @vec contains plain text of the latest
+ * version.
+ *
+ * Uptodate gaps of the @partial block with
+ * this plain text, encrypt the whole block
+ * and write the result to disk.
+ */
+static int32_t rmw_partial_block(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno,
+				 struct iovec *vec,
+				 int32_t count,
+				 struct iatt *stbuf,
+				 struct iobref *iobref,
+				 struct rmw_atom *atom)
+{
+	size_t was_read = 0;
+	uint64_t file_size;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	struct iovec *partial = atom->get_iovec(frame, 0);
+	struct avec_config *conf = atom->get_config(frame);
+	end_writeback_handler_t end_writeback_partial_block;
+#if DEBUG_CRYPT
+	gf_boolean_t check_last_cblock = _gf_false;
+#endif
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto exit;
+
+	file_size = local->cur_file_size;
+	was_read = op_ret;
+
+	if (atom->locality == HEAD_ATOM && conf->off_in_head) {
+		/*
+		 * head atom with a non-uptodate gap
+		 * at the beginning
+		 *
+		 * fill the gap with plain text of the
+		 * latest version. Convert a part of hole
+		 * (if any) to zeros.
+		 */
+		int32_t i;
+		int32_t copied = 0;
+		int32_t to_gap; /* amount of data needed to uptodate
+				   the gap at the beginning */
+#if 0
+		int32_t hole = 0; /* The part of the hole which
+				   * got in the head block */
+#endif /* 0 */
+		to_gap = conf->off_in_head;
+
+		if (was_read < to_gap) {
+			if (file_size >
+			    offset_at_head(conf) + was_read) {
+				/*
+				 * It is impossible to uptodate
+				 * head block: too few bytes have
+				 * been read from disk, so that
+				 * partial write is impossible.
+				 *
+				 * It could happen because of many
+				 * reasons: IO errors, (meta)data
+				 * corruption in the local file system,
+				 * etc.
+				 */
+				gf_log(this->name, GF_LOG_WARNING,
+				     "Can not uptodate a gap at the beginning");
+				local->op_ret = -1;
+				local->op_errno = EIO;
+				goto exit;
+			}
+#if 0
+			hole = to_gap - was_read;
+#endif /* 0 */
+			to_gap = was_read;
+		}
+		/*
+		 * uptodate the gap at the beginning
+		 */
+		for (i = 0; i < count && copied < to_gap; i++) {
+			int32_t to_copy;
+
+			to_copy = vec[i].iov_len;
+			if (to_copy > to_gap - copied)
+				to_copy = to_gap - copied;
+
+			memcpy(partial->iov_base, vec[i].iov_base, to_copy);
+			copied += to_copy;
+		}
+#if 0
+		/*
+		 * If possible, convert part of the
+		 * hole, which got in the head block
+		 */
+		ret = TRY_LOCK(&local->hole_lock);
+		if (!ret) {
+			if (local->hole_handled)
+				/*
+				 * already converted by
+				 * crypt_writev_cbk()
+				 */
+				UNLOCK(&local->hole_lock);
+			else {
+				/*
+				 * convert the part of the hole
+				 * which got in the head block
+				 * to zeros.
+				 *
+				 * Update the orig_offset to make
+				 * sure writev_cbk() won't care
+				 * about this part of the hole.
+				 *
+				 */
+				memset(partial->iov_base + to_gap, 0, hole);
+
+				conf->orig_offset -= hole;
+				conf->orig_size += hole;
+				UNLOCK(&local->hole_lock);
+			}
+		}
+		else /*
+		      * conversion is being performed
+		      * by crypt_writev_cbk()
+		      */
+			;
+#endif /* 0 */
+	}
+	if (atom->locality == TAIL_ATOM ||
+	    (!has_tail_block(conf) && conf->off_in_tail)) {
+		/*
+		 * tail atom, or head atom with a non-uptodate
+		 * gap at the end.
+		 *
+		 * fill the gap at the end of the block
+		 * with plain text of the latest version.
+		 * Pad the result, (if needed)
+		 */
+		int32_t i;
+		int32_t to_gap;
+		int copied;
+		off_t off_in_tail;
+		int32_t to_copy;
+
+		off_in_tail = conf->off_in_tail;
+		to_gap = conf->gap_in_tail;
+
+		if (to_gap && was_read < off_in_tail + to_gap) {
+			/*
+			 * It is impossible to uptodate
+			 * the gap at the end: too few bytes
+			 * have been read from disk, so that
+			 * partial write is impossible.
+			 *
+			 * It could happen because of many
+			 * reasons: IO errors, (meta)data
+			 * corruption in the local file system,
+			 * etc.
+			 */
+			gf_log(this->name, GF_LOG_WARNING,
+			       "Can not uptodate a gap at the end");
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+		/*
+		 * uptodate the gap at the end
+		 */
+		copied = 0;
+		to_copy = to_gap;
+		for(i = count - 1; i >= 0 && to_copy > 0; i--) {
+			uint32_t from_vec, off_in_vec;
+
+			off_in_vec = 0;
+			from_vec = vec[i].iov_len;
+			if (from_vec > to_copy) {
+				off_in_vec = from_vec - to_copy;
+				from_vec = to_copy;
+			}
+			memcpy(partial->iov_base +
+			       off_in_tail + to_gap - copied - from_vec,
+			       vec[i].iov_base + off_in_vec,
+			       from_vec);
+
+			gf_log(this->name, GF_LOG_DEBUG,
+			       "uptodate %d bytes at tail. Offset at target(source): %d(%d)",
+			       (int)from_vec,
+			       (int)off_in_tail + to_gap - copied - from_vec,
+			       (int)off_in_vec);
+
+			copied += from_vec;
+			to_copy -= from_vec;
+		}
+		partial->iov_len = off_in_tail + to_gap;
+
+		if (object_alg_should_pad(object)) {
+			int32_t resid = 0;
+			resid = partial->iov_len & (object_alg_blksize(object) - 1);
+			if (resid) {
+				/*
+				 * append a new EOF padding
+				 */
+				local->eof_padding_size =
+					object_alg_blksize(object) - resid;
+
+				gf_log(this->name, GF_LOG_DEBUG,
+				       "set padding size %d",
+				       local->eof_padding_size);
+
+				memset(partial->iov_base + partial->iov_len,
+				       1,
+				       local->eof_padding_size);
+				partial->iov_len += local->eof_padding_size;
+#if DEBUG_CRYPT
+				gf_log(this->name, GF_LOG_DEBUG,
+				       "pad cblock with %d zeros:",
+				       local->eof_padding_size);
+				dump_cblock(this,
+					    (unsigned char *)partial->iov_base +
+					    partial->iov_len - object_alg_blksize(object));
+				check_last_cblock = _gf_true;
+#endif
+			}
+		}
+	}
+	/*
+	 * encrypt the whole block
+	 */
+	encrypt_aligned_iov(object,
+			    partial,
+			    1,
+			    atom->offset_at(frame, object));
+#if DEBUG_CRYPT
+	if (check_last_cblock == _gf_true) {
+		gf_log(this->name, GF_LOG_DEBUG,
+		       "encrypt last cblock with offset %llu",
+		       (unsigned long long)atom->offset_at(frame, object));
+		dump_cblock(this, (unsigned char *)partial->iov_base +
+			    partial->iov_len - object_alg_blksize(object));
+	}
+#endif
+	set_local_io_params_writev(frame, object, atom,
+				   atom->offset_at(frame, object),
+				   iovec_get_size(partial, 1));
+	/*
+	 * write the whole block to disk
+	 */
+	end_writeback_partial_block = dispatch_end_writeback(local->fop);
+	conf->cursor ++;
+	STACK_WIND(frame,
+		   end_writeback_partial_block,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->writev,
+		   local->fd,
+		   partial,
+		   1,
+		   atom->offset_at(frame, object),
+		   local->flags,
+		   local->iobref_data,
+		   local->xdata);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "submit partial block: %d bytes from %d offset",
+	       (int)iovec_get_size(partial, 1),
+	       (int)atom->offset_at(frame, object));
+ exit:
+	return 0;
+}
+
+/*
+ * Perform a (read-)modify-write sequence.
+ * This should be performed only after approval
+ * of upper server-side manager, i.e. the caller
+ * needs to make sure this is his turn to rmw.
+ */
+void submit_partial(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    atom_locality_type ltype)
+{
+	int32_t ret;
+	dict_t *dict;
+	struct rmw_atom *atom;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	atom = atom_by_types(local->active_setup, ltype);
+	/*
+	 * To perform the "read" component of the read-modify-write
+	 * sequence the crypt translator does stack_wind to itself.
+	 * 
+	 * Pass current file size to crypt_readv()
+	 */
+	dict = dict_new();
+	if (!dict) {
+		/*
+		 * FIXME: Handle the error
+		 */
+		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
+		return;
+	}
+	ret = dict_set(dict,
+		       FSIZE_XATTR_PREFIX,
+		       data_from_uint64(local->cur_file_size));
+	if (ret) {
+		/*
+		 * FIXME: Handle the error
+		 */
+		dict_unref(dict);
+		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   atom->rmw,
+		   this,
+		   this->fops->readv, /* crypt_readv */
+		   fd,
+		   atom->count_to_uptodate(frame, object), /* count */
+		   atom->offset_at(frame, object), /* offset to read from */
+		   0,
+		   dict);
+ exit:
+	dict_unref(dict);
+}
+
+/*
+ * submit blocks of FULL_ATOM type
+ */
+void submit_full(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	struct rmw_atom *atom = atom_by_types(local->active_setup, FULL_ATOM);
+	uint32_t count; /* total number of full blocks to submit */
+	uint32_t granularity; /* number of blocks to submit in one iteration */
+
+	uint64_t off_in_file; /* start offset in the file, bytes */
+	uint32_t off_in_atom; /* start offset in the atom, blocks */
+	uint32_t blocks_written = 0; /* blocks written for this submit */
+
+	struct avec_config *conf = atom->get_config(frame);
+	end_writeback_handler_t end_writeback_full_block;
+	/*
+	 * Write full blocks by groups of granularity size.
+	 */
+	end_writeback_full_block = dispatch_end_writeback(local->fop);
+
+	if (is_ordered_mode(frame)) {
+		uint32_t skip = has_head_block(conf) ? 1 : 0;
+		count = 1;
+		granularity = 1;
+		/*
+		 * calculate start offset using cursor value;
+		 * here we should take into accout head block,
+		 * which corresponds to cursor value 0.
+		 */
+		off_in_file = atom->offset_at(frame, object) +
+			((conf->cursor - skip) << get_atom_bits(object));
+		off_in_atom = conf->cursor - skip;
+	}
+	else {
+		/*
+		 * in parallel mode
+		 */
+		count = conf->nr_full_blocks;
+		granularity = MAX_IOVEC;
+		off_in_file = atom->offset_at(frame, object);
+		off_in_atom = 0;
+	}
+	while (count) {
+		uint32_t blocks_to_write = count;
+
+		if (blocks_to_write > granularity)
+			blocks_to_write = granularity;
+		if (conf->type == HOLE_ATOM)
+			/*
+			 * reset iovec before encryption
+			 */
+			memset(atom->get_iovec(frame, 0)->iov_base,
+			       0,
+			       get_atom_size(object));
+		/*
+		 * encrypt the group
+		 */
+		encrypt_aligned_iov(object,
+				    atom->get_iovec(frame,
+						    off_in_atom +
+						    blocks_written),
+				    blocks_to_write,
+				    off_in_file + (blocks_written <<
+						   get_atom_bits(object)));
+		
+		set_local_io_params_writev(frame, object, atom,
+		        off_in_file + (blocks_written << get_atom_bits(object)),
+			blocks_to_write <<  get_atom_bits(object));
+
+		conf->cursor += blocks_to_write;
+
+		STACK_WIND(frame,
+			   end_writeback_full_block,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->writev,
+			   local->fd,
+			   atom->get_iovec(frame, off_in_atom + blocks_written),
+			   blocks_to_write,
+			   off_in_file + (blocks_written << get_atom_bits(object)),
+			   local->flags,
+			   local->iobref_data ? local->iobref_data : local->iobref,
+			   local->xdata);
+
+		gf_log("crypt", GF_LOG_DEBUG, "submit %d full blocks from %d offset",
+		       blocks_to_write,
+		       (int)(off_in_file + (blocks_written << get_atom_bits(object))));
+
+		count -= blocks_to_write;
+		blocks_written += blocks_to_write;
+	}
+	return;
+}
+
+static int32_t rmw_data_head(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(DATA_ATOM, HEAD_ATOM));
+}
+
+static int32_t rmw_data_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(DATA_ATOM, TAIL_ATOM));
+}
+
+static int32_t rmw_hole_head(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(HOLE_ATOM, HEAD_ATOM));
+}
+
+static int32_t rmw_hole_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(HOLE_ATOM, TAIL_ATOM));
+}
+
+/*
+ * atom->count_to_uptodate()
+ */
+static uint32_t count_to_uptodate_head(struct avec_config *conf,
+				       struct object_cipher_info *object)
+{
+	if (conf->acount == 1 && conf->off_in_tail)
+		return get_atom_size(object);
+	else
+		/* there is no need to read the whole head block */
+		return conf->off_in_head;
+}
+
+static uint32_t count_to_uptodate_tail(struct avec_config *conf,
+				       struct object_cipher_info *object)
+{
+	/* we need to read the whole tail block */
+	return get_atom_size(object);
+}
+
+static uint32_t count_to_uptodate_data_head(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_head(get_data_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_data_tail(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_tail(get_data_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_hole_head(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_head(get_hole_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_hole_tail(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_tail(get_hole_conf(frame), object);
+}
+
+/* atom->get_config() */
+
+static struct avec_config *get_config_data(call_frame_t *frame)
+{
+	return &((crypt_local_t *)frame->local)->data_conf;
+}
+
+static struct avec_config *get_config_hole(call_frame_t *frame)
+{
+	return &((crypt_local_t *)frame->local)->hole_conf;
+}
+
+/*
+ * atom->get_iovec()
+ */
+static struct iovec *get_iovec_hole_head(call_frame_t *frame,
+					 uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec;
+}
+
+static struct iovec *get_iovec_hole_full(call_frame_t *frame,
+					 uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec + (conf->off_in_head ? 1 : 0);
+}
+
+static inline struct iovec *get_iovec_hole_tail(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec + (conf->blocks_in_pool - 1);
+}
+
+static inline struct iovec *get_iovec_data_head(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec;
+}
+
+static inline struct iovec *get_iovec_data_full(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec + (conf->off_in_head ? 1 : 0) + count;
+}
+
+static inline struct iovec *get_iovec_data_tail(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec +
+		(conf->off_in_head ? 1 : 0) +
+		conf->nr_full_blocks;
+}
+
+static struct rmw_atom atoms[LAST_DATA_TYPE][LAST_LOCALITY_TYPE] = {
+	[DATA_ATOM][HEAD_ATOM] =
+	{ .locality = HEAD_ATOM,
+	  .rmw = rmw_data_head,
+	  .offset_at = offset_at_data_head,
+	  .offset_in = offset_in_data_head,
+	  .get_iovec = get_iovec_data_head,
+	  .io_size_nopad = io_size_nopad_data_head,
+	  .count_to_uptodate = count_to_uptodate_data_head,
+	  .get_config = get_config_data
+	},
+	[DATA_ATOM][TAIL_ATOM] =
+	{ .locality = TAIL_ATOM,
+	  .rmw = rmw_data_tail,
+	  .offset_at = offset_at_data_tail,
+	  .offset_in = offset_in_tail,
+	  .get_iovec = get_iovec_data_tail,
+	  .io_size_nopad = io_size_nopad_data_tail,
+	  .count_to_uptodate = count_to_uptodate_data_tail,
+	  .get_config = get_config_data
+	},
+	[DATA_ATOM][FULL_ATOM] =
+	{ .locality = FULL_ATOM,
+	  .offset_at = offset_at_data_full,
+	  .offset_in = offset_in_data_full,
+	  .get_iovec = get_iovec_data_full,
+	  .io_size_nopad = io_size_nopad_data_full,
+	  .get_config = get_config_data
+	},
+	[HOLE_ATOM][HEAD_ATOM] =
+	{ .locality = HEAD_ATOM,
+	  .rmw = rmw_hole_head,
+	  .offset_at = offset_at_hole_head,
+	  .offset_in = offset_in_hole_head,
+	  .get_iovec = get_iovec_hole_head,
+	  .io_size_nopad = io_size_nopad_hole_head,
+	  .count_to_uptodate = count_to_uptodate_hole_head,
+	  .get_config = get_config_hole
+	},
+	[HOLE_ATOM][TAIL_ATOM] =
+	{ .locality = TAIL_ATOM,
+	  .rmw = rmw_hole_tail,
+	  .offset_at = offset_at_hole_tail,
+	  .offset_in = offset_in_tail,
+	  .get_iovec = get_iovec_hole_tail,
+	  .io_size_nopad = io_size_nopad_hole_tail,
+	  .count_to_uptodate = count_to_uptodate_hole_tail,
+	  .get_config = get_config_hole
+	},
+	[HOLE_ATOM][FULL_ATOM] =
+	{ .locality = FULL_ATOM,
+	  .offset_at = offset_at_hole_full,
+	  .offset_in = offset_in_hole_full,
+	  .get_iovec = get_iovec_hole_full,
+	  .io_size_nopad = io_size_nopad_hole_full,
+	  .get_config = get_config_hole
+	}
+};
+
+struct rmw_atom *atom_by_types(atom_data_type data,
+			       atom_locality_type locality)
+{
+	return &atoms[data][locality];
+}
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt-common.h b/xlators/encryption/crypt/src/crypt-common.h
new file mode 100644
index 000000000..7c212ad5d
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt-common.h
@@ -0,0 +1,141 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CRYPT_COMMON_H__
+#define __CRYPT_COMMON_H__
+
+#define INVAL_SUBVERSION_NUMBER (0xff)
+#define CRYPT_INVAL_OP   (GF_FOP_NULL)
+
+#define CRYPTO_FORMAT_PREFIX  "trusted.glusterfs.crypt.att.cfmt"
+#define FSIZE_XATTR_PREFIX    "trusted.glusterfs.crypt.att.size"
+#define SUBREQ_PREFIX         "trusted.glusterfs.crypt.msg.sreq"
+#define FSIZE_MSG_PREFIX      "trusted.glusterfs.crypt.msg.size"
+#define DE_MSG_PREFIX         "trusted.glusterfs.crypt.msg.dent"
+#define REQUEST_ID_PREFIX     "trusted.glusterfs.crypt.msg.rqid"
+#define MSGFLAGS_PREFIX       "trusted.glusterfs.crypt.msg.xfgs"
+
+
+/* messages for crypt_open() */
+#define MSGFLAGS_REQUEST_MTD_RLOCK   1 /* take read lock and don't unlock */
+#define MSGFLAGS_REQUEST_MTD_WLOCK   2 /* take write lock and don't unlock */
+
+#define AES_BLOCK_BITS (4) /* AES_BLOCK_SIZE == 1 << AES_BLOCK_BITS */
+
+#define noop   do {; } while (0)
+#define cassert(cond) ({ switch (-1) { case (cond): case 0: break; } })
+#define __round_mask(x, y) ((__typeof__(x))((y)-1))
+#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
+
+/*
+ * Format of file's metadata
+ */
+struct crypt_format {
+	uint8_t loader_id;  /* version of metadata loader */
+	uint8_t versioned[0]; /* file's metadata of specific version */
+} __attribute__((packed));
+
+typedef enum {
+	AES_CIPHER_ALG,
+	LAST_CIPHER_ALG
+} cipher_alg_t;
+
+typedef enum {
+	XTS_CIPHER_MODE,
+	LAST_CIPHER_MODE
+} cipher_mode_t;
+
+typedef enum {
+	MTD_LOADER_V1,
+	LAST_MTD_LOADER
+} mtd_loader_id;
+
+static inline void msgflags_set_mtd_rlock(uint32_t *flags)
+{
+	*flags |= MSGFLAGS_REQUEST_MTD_RLOCK;
+}
+
+static inline void msgflags_set_mtd_wlock(uint32_t *flags)
+{
+	*flags |= MSGFLAGS_REQUEST_MTD_WLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_rlock(uint32_t *flags)
+{
+	return *flags & MSGFLAGS_REQUEST_MTD_RLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_wlock(uint32_t *flags)
+{
+	return *flags & MSGFLAGS_REQUEST_MTD_WLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_lock(uint32_t *flags)
+{
+	return  msgflags_check_mtd_rlock(flags) ||
+		msgflags_check_mtd_wlock(flags);
+}
+
+/*
+ * returns number of logical blocks occupied
+ * (maybe partially) by @count bytes
+ * at offset @start.
+ */
+static inline off_t logical_blocks_occupied(uint64_t start, off_t count,
+					    int blkbits)
+{
+	return ((start + count - 1) >> blkbits) - (start >> blkbits) + 1;
+}
+
+/*
+ * are two bytes (represented by offsets @off1
+ * and @off2 respectively) in the same logical
+ * block.
+ */
+static inline int in_same_lblock(uint64_t off1, uint64_t off2,
+				 int blkbits)
+{
+	return off1 >> blkbits == off2 >> blkbits;
+}
+
+static inline void dump_cblock(xlator_t *this, unsigned char *buf)
+{
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "dump cblock: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x",
+	       (buf)[0],
+	       (buf)[1],
+	       (buf)[2],
+	       (buf)[3],
+	       (buf)[4],
+	       (buf)[5],
+	       (buf)[6],
+	       (buf)[7],
+	       (buf)[8],
+	       (buf)[9],
+	       (buf)[10],
+	       (buf)[11],
+	       (buf)[12],
+	       (buf)[13],
+	       (buf)[14],
+	       (buf)[15]);
+}
+
+#endif /* __CRYPT_COMMON_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt-mem-types.h b/xlators/encryption/crypt/src/crypt-mem-types.h
new file mode 100644
index 000000000..799727573
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt-mem-types.h
@@ -0,0 +1,43 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __CRYPT_MEM_TYPES_H__
+#define __CRYPT_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_crypt_mem_types_ {
+	gf_crypt_mt_priv = gf_common_mt_end + 1,
+	gf_crypt_mt_inode,
+	gf_crypt_mt_data,
+	gf_crypt_mt_mtd,
+	gf_crypt_mt_loc,
+	gf_crypt_mt_iatt,
+	gf_crypt_mt_key,
+	gf_crypt_mt_iovec,
+	gf_crypt_mt_char,
+};
+
+#endif /* __CRYPT_MEM_TYPES_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
+
+
+
diff --git a/xlators/encryption/crypt/src/crypt.c b/xlators/encryption/crypt/src/crypt.c
new file mode 100644
index 000000000..db2e6d83c
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt.c
@@ -0,0 +1,4498 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#include <ctype.h>
+#include <sys/uio.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "defaults.h"
+
+#include "crypt-common.h"
+#include "crypt.h"
+
+static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd);
+static int32_t init_inode_info_tail(struct crypt_inode_info *info,
+				    struct master_cipher_info *master);
+static int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
+				       uint64_t from, off_t size);
+static int32_t load_file_size(call_frame_t *frame, void *cookie,
+			      xlator_t *this, int32_t op_ret, int32_t op_errno,
+			      dict_t *dict, dict_t *xdata);
+static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
+			      atom_data_type dtype);
+static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
+			       atom_data_type dtype);
+static void put_one_call_open(call_frame_t *frame);
+static void put_one_call_readv(call_frame_t *frame, xlator_t *this);
+static void put_one_call_writev(call_frame_t *frame, xlator_t *this);
+static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this);
+static void free_avec(struct iovec *avec, char **pool, int blocks_in_pool);
+static void free_avec_data(crypt_local_t *local);
+static void free_avec_hole(crypt_local_t *local);
+
+static crypt_local_t *crypt_alloc_local(call_frame_t *frame, xlator_t *this,
+					glusterfs_fop_t fop)
+{
+	crypt_local_t *local = NULL;
+
+	local = mem_get0(this->local_pool);
+	if (!local) {
+		gf_log(this->name, GF_LOG_ERROR, "out of memory");
+		return NULL;
+	}
+	local->fop = fop;
+	LOCK_INIT(&local->hole_lock);
+	LOCK_INIT(&local->call_lock);
+	LOCK_INIT(&local->rw_count_lock);
+
+	frame->local = local;
+	return local;
+}
+
+struct crypt_inode_info *get_crypt_inode_info(inode_t *inode, xlator_t *this)
+{
+	int ret;
+	uint64_t value = 0;
+	struct crypt_inode_info *info;
+
+	ret = inode_ctx_get(inode, this, &value);
+	if (ret == -1) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"Can not get inode info");
+		return NULL;
+	}
+	info = (struct crypt_inode_info *)(long)value;
+	if (info == NULL) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"Can not obtain inode info");
+		return NULL;
+	}
+	return info;
+}
+
+static struct crypt_inode_info *local_get_inode_info(crypt_local_t *local,
+						      xlator_t *this)
+{
+	if (local->info)
+		return local->info;
+	local->info = get_crypt_inode_info(local->fd->inode, this);
+	return local->info;
+}
+
+static struct crypt_inode_info *alloc_inode_info(crypt_local_t *local,
+						 loc_t *loc)
+{
+	struct crypt_inode_info *info;
+
+	info = GF_CALLOC(1, sizeof(*info), gf_crypt_mt_inode);
+	if (!info) {
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+		gf_log ("crypt", GF_LOG_WARNING,
+			"Can not allocate inode info");
+		return NULL;
+	}
+	memset(info, 0, sizeof(*info));
+#if DEBUG_CRYPT
+	info->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!info->loc) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not allocate loc");
+		GF_FREE(info);
+		return NULL;
+	}
+	if (loc_copy(info->loc, loc)){
+		GF_FREE(info->loc);
+		GF_FREE(info);
+		return NULL;
+	}
+#endif /* DEBUG_CRYPT */
+
+	local->info = info;
+	return info;
+}
+
+static void free_inode_info(struct crypt_inode_info *info)
+{
+#if DEBUG_CRYPT
+	loc_wipe(info->loc);
+	GF_FREE(info->loc);
+#endif
+	memset(info, 0, sizeof(*info));
+	GF_FREE(info);
+}
+
+int crypt_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t ctx_addr = 0;
+        if (!inode_ctx_del (inode, this, &ctx_addr))
+                free_inode_info((struct crypt_inode_info *)(long)ctx_addr);
+        return 0;
+}
+
+#if DEBUG_CRYPT
+static void check_read(call_frame_t *frame, xlator_t *this, int32_t read,
+		       struct iovec *vec, int32_t count, struct iatt *stbuf)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = get_object_cinfo(local->info);
+	struct avec_config *conf = &local->data_conf;
+	uint32_t resid = stbuf->ia_size & (object_alg_blksize(object) - 1);
+
+	if (read <= 0)
+		return;
+	if (read != iovec_get_size(vec, count))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"op_ret differs from amount of read bytes");
+
+	if (object_alg_should_pad(object) && (read & (object_alg_blksize(object) - 1)))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad amount of read bytes (!= 0 mod(cblock size))");
+
+	if (conf->aligned_offset + read >
+	    stbuf->ia_size + (resid ? object_alg_blksize(object) - resid : 0))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad amount of read bytes (too large))");
+
+}
+
+#define PT_BYTES_TO_DUMP (32)
+static void dump_plain_text(crypt_local_t *local, struct iovec *avec)
+{
+	int32_t to_dump;
+	char str[PT_BYTES_TO_DUMP + 1];
+
+	if (!avec)
+		return;
+	to_dump = avec->iov_len;
+	if (to_dump > PT_BYTES_TO_DUMP)
+		to_dump = PT_BYTES_TO_DUMP;
+	memcpy(str, avec->iov_base, to_dump);
+	memset(str + to_dump, '0', 1);
+	gf_log("crypt", GF_LOG_DEBUG, "Read file: %s", str);
+}
+
+static int32_t data_conf_invariant(struct avec_config *conf)
+{
+	return conf->acount ==
+		!!has_head_block(conf) +
+		!!has_tail_block(conf)+
+		conf->nr_full_blocks;
+}
+
+static int32_t hole_conf_invariant(struct avec_config *conf)
+{
+	return conf->blocks_in_pool ==
+		!!has_head_block(conf) +
+		!!has_tail_block(conf)+
+		!!has_full_blocks(conf);
+}
+
+static void crypt_check_conf(struct avec_config *conf)
+{
+	int32_t ret = 0;
+	const char *msg;
+
+	switch (conf->type) {
+	case DATA_ATOM:
+		msg = "data";
+		ret = data_conf_invariant(conf);
+		break;
+	case HOLE_ATOM:
+		msg = "hole";
+		ret = hole_conf_invariant(conf);
+		break;
+	default:
+		msg = "unknown";
+	}
+	if (!ret)
+		gf_log("crypt", GF_LOG_DEBUG, "bad %s conf", msg);
+}
+
+static void check_buf(call_frame_t *frame, xlator_t *this, struct iatt *buf)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	uint64_t local_file_size;
+
+	switch(local->fop) {
+	case GF_FOP_FTRUNCATE:
+		return;
+	case GF_FOP_WRITE:
+		local_file_size = local->new_file_size;
+		break;
+	case GF_FOP_READ:
+		if (parent_is_crypt_xlator(frame, this))
+			return;
+		local_file_size = local->cur_file_size;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad file operation");
+		return;
+	}
+	if (buf->ia_size != round_up(local_file_size,
+					 object_alg_blksize(object)))
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "bad ia_size in buf (%llu), should be %llu",
+		       (unsigned long long)buf->ia_size,
+		       (unsigned long long)round_up(local_file_size,
+						    object_alg_blksize(object)));
+}
+
+#else
+#define check_read(frame, this, op_ret, vec, count, stbuf) noop
+#define dump_plain_text(local, avec) noop
+#define crypt_check_conf(conf) noop
+#define check_buf(frame, this, buf) noop
+#endif /* DEBUG_CRYPT */
+
+/*
+ * Pre-conditions:
+ * @vec represents a ciphertext of expanded size and
+ * aligned offset.
+ *
+ * Compound a temporal vector @avec with block-aligned
+ * components, decrypt and fix it up to represent a chunk
+ * of data corresponding to the original size and offset.
+ * Pass the result to the next translator.
+ */
+int32_t crypt_readv_cbk(call_frame_t *frame,
+			void *cookie,
+			xlator_t *this,
+			int32_t op_ret,
+			int32_t op_errno,
+			struct iovec *vec,
+			int32_t count,
+			struct iatt *stbuf,
+			struct iobref *iobref,
+			dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	struct iovec *avec;
+	uint32_t i;
+	uint32_t to_vec;
+	uint32_t to_user;
+
+	check_buf(frame, this, stbuf);
+	check_read(frame, this, op_ret, vec, count, stbuf);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->iobref = iobref_ref(iobref);
+
+	local->buf = *stbuf;
+	local->buf.ia_size = local->cur_file_size;
+
+	if (op_ret <= 0 || count == 0 || vec[0].iov_len == 0)
+		goto put_one_call;
+
+	if (conf->orig_offset >= local->cur_file_size) {
+		local->op_ret = 0;
+		goto put_one_call;
+	}
+	/* 
+	 * correct config params with real file size
+	 * and actual amount of bytes read
+	 */
+	set_config_offsets(frame, this,
+			   conf->orig_offset, op_ret, DATA_ATOM, 0);
+
+	if (conf->orig_offset + conf->orig_size > local->cur_file_size)
+		conf->orig_size = local->cur_file_size - conf->orig_offset;
+	/*
+	 * calculate amount of data to be returned
+	 * to user.
+	 */
+	to_user = op_ret;
+	if (conf->aligned_offset + to_user <= conf->orig_offset) {
+		gf_log(this->name, GF_LOG_WARNING, "Incomplete read");
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto put_one_call;
+	}
+	to_user -= (conf->aligned_offset - conf->orig_offset);
+
+	if (to_user > conf->orig_size)
+		to_user = conf->orig_size;
+	local->rw_count = to_user;
+
+	op_errno = set_config_avec_data(this, local,
+					conf, object, vec, count);
+	if (op_errno) {
+		local->op_ret = -1;
+		local->op_errno = op_errno;
+		goto put_one_call;
+	}
+	avec = conf->avec;
+#if DEBUG_CRYPT
+	if (conf->off_in_tail != 0 &&
+	    conf->off_in_tail < object_alg_blksize(object) &&
+	    object_alg_should_pad(object))
+		gf_log(this->name, GF_LOG_DEBUG, "Bad offset in tail %d",
+		       conf->off_in_tail);
+	if (iovec_get_size(vec, count) != 0 &&
+	    in_same_lblock(conf->orig_offset + iovec_get_size(vec, count) - 1,
+			   local->cur_file_size - 1,
+			   object_alg_blkbits(object))) {
+		gf_log(this->name, GF_LOG_DEBUG, "Compound last cblock");
+		dump_cblock(this,
+			    (unsigned char *)(avec[conf->acount - 1].iov_base) +
+			    avec[conf->acount - 1].iov_len - object_alg_blksize(object));
+		dump_cblock(this,
+			    (unsigned char *)(vec[count - 1].iov_base) +
+			    vec[count - 1].iov_len - object_alg_blksize(object));
+	}
+#endif
+	decrypt_aligned_iov(object, avec,
+			    conf->acount, conf->aligned_offset);
+	/*
+	 * pass proper plain data to user
+	 */
+	avec[0].iov_base += (conf->aligned_offset - conf->orig_offset);
+	avec[0].iov_len  -= (conf->aligned_offset - conf->orig_offset);
+
+	to_vec = to_user;
+	for (i = 0; i < conf->acount; i++) {
+		if (avec[i].iov_len > to_vec)
+			avec[i].iov_len = to_vec;
+		to_vec -= avec[i].iov_len;
+	}
+ put_one_call:
+	put_one_call_readv(frame, this);
+	return 0;
+}
+
+static int32_t do_readv(call_frame_t *frame,
+			void *cookie,
+			xlator_t *this,
+			int32_t op_ret,
+			int32_t op_errno,
+			dict_t *dict,
+			dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;	
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_errno = EIO;
+		goto error;
+	}
+	local->cur_file_size = data_to_uint64(data);
+	
+	get_one_call(frame);
+	STACK_WIND(frame,
+		   crypt_readv_cbk,
+		   FIRST_CHILD (this),
+		   FIRST_CHILD (this)->fops->readv,
+		   local->fd,
+		   /*
+		    * FIXME: read amount can be reduced
+		    */
+		   local->data_conf.expanded_size,
+		   local->data_conf.aligned_offset,
+		   local->flags,
+		   local->xdata);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+
+	get_one_call(frame);
+	put_one_call_readv(frame, this);
+	return 0;
+}
+
+static int32_t crypt_readv_finodelk_cbk(call_frame_t *frame,
+					void *cookie,
+					xlator_t *this,
+					int32_t op_ret,
+					int32_t op_errno,
+					dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size
+	 */
+	STACK_WIND(frame,
+		   do_readv,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return  0;
+ error:
+	fd_unref(local->fd);
+	if (local->xdata)
+		dict_unref(local->xdata);
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    -1,
+			    op_errno,
+			    NULL,
+			    0,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t readv_trivial_completion(call_frame_t *frame,
+					void *cookie,
+					xlator_t *this,
+					int32_t op_ret,
+					int32_t op_errno,
+					struct iatt *buf,
+					dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "stat failed (%d)", op_errno);
+		goto error;
+	}
+	local->buf = *buf;
+	STACK_WIND(frame,
+		   load_file_size,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->getxattr,
+		   local->loc,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;	
+ error:
+	STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno,
+			    NULL, 0, NULL, NULL, NULL);
+	return 0;
+}
+
+int32_t crypt_readv(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    size_t size,
+		    off_t offset,
+		    uint32_t flags, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+
+#if DEBUG_CRYPT
+	gf_log("crypt", GF_LOG_DEBUG, "reading %d bytes from offset %llu",
+	       (int)size, (long long)offset);
+	if (parent_is_crypt_xlator(frame, this))
+		gf_log("crypt", GF_LOG_DEBUG, "parent is crypt");
+#endif	
+	local = crypt_alloc_local(frame, this, GF_FOP_READ);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	if (size == 0)
+		goto trivial;
+
+	local->fd = fd_ref(fd);
+	local->flags = flags;
+
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		fd_unref(fd);
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		fd_unref(fd);
+		goto error;
+	}
+	set_config_offsets(frame, this, offset, size,
+			   DATA_ATOM, 0);
+	if (parent_is_crypt_xlator(frame, this)) {
+		data_t *data;
+		/*
+		 * We are called by crypt_writev (or cypt_ftruncate)
+		 * to perform the "read" component of the read-modify-write
+		 * (or read-prune-write) sequence for some atom;
+		 *
+		 * don't ask for access:
+		 * it has already been acquired
+		 *
+		 * Retrieve current file size
+		 */
+		if (!xdata) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size hasn't been passed");
+			ret = EIO;
+			goto error;
+		}
+		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
+		if (!data) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size not found");
+			ret = EIO;
+			goto error;
+		}
+		local->old_file_size =
+			local->cur_file_size = data_to_uint64(data);
+
+		get_one_call(frame);
+		STACK_WIND(frame,
+			   crypt_readv_cbk,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->readv,
+			   local->fd,
+			   /*
+			    * FIXME: read amount can be reduced
+			    */
+			   local->data_conf.expanded_size,
+			   local->data_conf.aligned_offset,
+			   flags,
+			   NULL);
+		return 0;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_RDLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_readv_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   readv_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    -1,
+			    ret,
+			    NULL,
+			    0,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+void set_local_io_params_writev(call_frame_t *frame,
+				struct object_cipher_info *object,
+				struct rmw_atom *atom,
+				off_t io_offset,
+				uint32_t io_size)
+{
+	crypt_local_t *local = frame->local;
+
+	local->io_offset = io_offset;
+	local->io_size = io_size;
+
+	local->io_offset_nopad =
+		atom->offset_at(frame, object) + atom->offset_in(frame, object);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "set nopad offset to %llu",
+	       (unsigned long long)local->io_offset_nopad);
+
+	local->io_size_nopad = atom->io_size_nopad(frame, object);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "set nopad size to %llu",
+	       (unsigned long long)local->io_size_nopad);
+
+	local->update_disk_file_size = 0;
+	/*
+	 * NOTE: eof_padding_size is 0 for all full atoms;
+	 * For head and tail atoms it will be set up at rmw_partial block()
+	 */
+	local->new_file_size = local->cur_file_size;
+
+	if (local->io_offset_nopad + local->io_size_nopad > local->cur_file_size) {
+
+		local->new_file_size = local->io_offset_nopad + local->io_size_nopad;
+
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "set new file size to %llu",
+		       (unsigned long long)local->new_file_size);
+
+		local->update_disk_file_size = 1;
+	}
+}
+
+void set_local_io_params_ftruncate(call_frame_t *frame,
+				   struct object_cipher_info *object)
+{
+	uint32_t resid;
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	resid = conf->orig_offset & (object_alg_blksize(object) - 1);
+	if (resid) {
+		local->eof_padding_size =
+			object_alg_blksize(object) - resid;
+		local->new_file_size = conf->aligned_offset;
+		local->update_disk_file_size = 0;
+		/*
+		 * file size will be updated
+		 * in the ->writev() stack,
+		 * when submitting file tail
+		 */
+	}
+	else {	
+		local->eof_padding_size = 0;
+		local->new_file_size = conf->orig_offset;
+		local->update_disk_file_size = 1;
+		/*
+		 * file size will be updated
+		 * in this ->ftruncate stack
+		 */
+	}
+}
+
+static inline void submit_head(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	submit_partial(frame, this, local->fd, HEAD_ATOM);
+}
+
+static inline void submit_tail(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	submit_partial(frame, this, local->fd, TAIL_ATOM);
+}
+
+static void submit_hole(call_frame_t *frame, xlator_t *this)
+{
+	/*
+	 * hole conversion always means
+	 * appended write and goes in ordered fashion
+	 */
+	do_ordered_submit(frame, this, HOLE_ATOM);
+}
+
+static void submit_data(call_frame_t *frame, xlator_t *this)
+{
+	if (is_ordered_mode(frame)) {
+		do_ordered_submit(frame, this, DATA_ATOM);
+		return;
+	}
+	gf_log("crypt", GF_LOG_WARNING, "Bad submit mode");
+	get_nr_calls(frame, nr_calls_data(frame));
+	do_parallel_submit(frame, this, DATA_ATOM);
+	return;
+}
+
+/*
+ * heplers called by writev_cbk, fruncate_cbk in ordered mode
+ */
+
+static inline int32_t should_submit_hole(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+
+	return conf->avec != NULL;
+}
+
+static inline int32_t should_resume_submit_hole(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+
+	if (local->fop == GF_FOP_WRITE && has_tail_block(conf))
+		/*
+		 * Don't submit a part of hole, which
+		 * fits into a data block:
+ 		 * this part of hole will be converted
+		 * as a gap filled by zeros in data head
+		 * block.
+		 */
+		return conf->cursor < conf->acount - 1;
+	else
+		return conf->cursor < conf->acount;
+}
+
+static inline int32_t should_resume_submit_data(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	if (is_ordered_mode(frame))
+		return conf->cursor < conf->acount;
+	/*
+	 * parallel writes
+	 */
+	return 0;
+}
+
+static inline int32_t should_submit_data_after_hole(crypt_local_t *local)
+{
+	return local->data_conf.avec != NULL;
+}
+
+static void update_local_file_params(call_frame_t *frame,
+				     xlator_t *this,
+				     struct iatt *prebuf,
+				     struct iatt *postbuf)
+{
+	crypt_local_t *local = frame->local;
+
+	check_buf(frame, this, postbuf);
+
+	local->prebuf = *prebuf;
+	local->postbuf = *postbuf;
+
+	local->prebuf.ia_size  = local->cur_file_size;
+	local->postbuf.ia_size = local->new_file_size;
+
+	local->cur_file_size = local->new_file_size;	
+}
+
+static int32_t end_writeback_writev(call_frame_t *frame,
+				    void *cookie,
+				    xlator_t *this,
+				    int32_t op_ret,
+				    int32_t op_errno,
+				    struct iatt *prebuf,
+				    struct iatt *postbuf,
+				    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret <= 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "writev iteration failed");
+		goto put_one_call;
+	}
+	/*
+	 * op_ret includes paddings (atom's head, atom's tail and EOF)
+	 */
+	if (op_ret < local->io_size) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Incomplete writev iteration");
+		goto put_one_call;
+	}
+	op_ret -= local->eof_padding_size;
+	local->op_ret = op_ret;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	if (data_write_in_progress(local)) {
+
+		LOCK(&local->rw_count_lock);
+		local->rw_count += op_ret;
+		UNLOCK(&local->rw_count_lock);
+
+		if (should_resume_submit_data(frame))
+			submit_data(frame, this);
+	}
+	else {
+		/*
+		 * hole conversion is going on;
+		 * don't take into account written zeros
+		 */
+		if (should_resume_submit_hole(local))
+			submit_hole(frame, this);
+
+		else if (should_submit_data_after_hole(local))
+			submit_data(frame, this);
+	}
+ put_one_call:
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+#define crypt_writev_cbk end_writeback_writev
+
+#define HOLE_WRITE_CHUNK_BITS 12
+#define HOLE_WRITE_CHUNK_SIZE (1 << HOLE_WRITE_CHUNK_BITS)
+
+/*
+ * Convert hole of size @size at offset @off to
+ * zeros and prepare respective iovecs for submit.
+ * The hole lock should be held.
+ *
+ * Pre-conditions:
+ * @local->file_size is set and valid.
+ */
+int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
+				uint64_t off, off_t size)
+{
+	int32_t ret;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_config_offsets(frame, this, off, size, HOLE_ATOM, 1);
+
+	ret = set_config_avec_hole(this, local,
+				   &local->hole_conf, object, local->fop);
+	crypt_check_conf(&local->hole_conf);
+
+	return ret;
+}
+
+/*
+ * prepare for submit @count bytes at offset @from
+ */
+int32_t prepare_for_submit_data(call_frame_t *frame, xlator_t *this,
+				off_t from, int32_t size, struct iovec *vec,
+				int32_t vec_count, int32_t setup_gap)
+{
+	uint32_t ret;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_config_offsets(frame, this, from, size,
+			   DATA_ATOM, setup_gap);
+
+	ret = set_config_avec_data(this, local,
+				   &local->data_conf, object, vec, vec_count);
+	crypt_check_conf(&local->data_conf);
+
+	return ret;
+}
+
+static void free_avec(struct iovec *avec,
+		      char **pool, int blocks_in_pool)
+{
+	if (!avec)
+		return;
+	GF_FREE(pool);
+	GF_FREE(avec);
+}
+
+static void free_avec_data(crypt_local_t *local)
+{
+	return free_avec(local->data_conf.avec,
+			 local->data_conf.pool,
+			 local->data_conf.blocks_in_pool);
+}
+
+static void free_avec_hole(crypt_local_t *local)
+{
+	return free_avec(local->hole_conf.avec,
+			 local->hole_conf.pool,
+			 local->hole_conf.blocks_in_pool);
+}
+
+
+static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
+			       atom_data_type dtype)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf;
+
+	local->active_setup = dtype;
+	conf = conf_by_type(frame, dtype);
+
+	if (has_head_block(conf))
+		submit_head(frame, this);
+
+	if (has_full_blocks(conf))
+		submit_full(frame, this);
+
+	if (has_tail_block(conf))
+		submit_tail(frame, this);
+	return;
+}
+
+static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
+			      atom_data_type dtype)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf;
+
+	local->active_setup = dtype;
+	conf = conf_by_type(frame, dtype);
+
+	if (should_submit_head_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_head(frame, this);
+	}
+	else if (should_submit_full_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_full(frame, this);
+	}
+	else if (should_submit_tail_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_tail(frame, this);
+	}
+	else
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "nothing has been submitted in ordered mode");
+	return;
+}
+
+static int32_t do_writev(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *dict,
+			 dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_ret = -1;
+		op_errno = EIO;
+		goto error;
+	}
+	local->old_file_size = local->cur_file_size = data_to_uint64(data);
+
+	set_gap_at_end(frame, object, &local->data_conf, DATA_ATOM);
+
+	if (local->cur_file_size < local->data_conf.orig_offset) {
+		/*
+		 * Set up hole config
+		 */
+		op_errno = prepare_for_submit_hole(frame,
+			       this,
+			       local->cur_file_size,
+			       local->data_conf.orig_offset - local->cur_file_size);
+		if (op_errno) {
+			local->op_ret = -1;
+			local->op_errno = op_errno;
+			goto error;
+		}
+	}
+	if (should_submit_hole(local))
+		submit_hole(frame, this);
+	else 
+		submit_data(frame, this);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+static int32_t crypt_writev_finodelk_cbk(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size first
+	 */
+	STACK_WIND(frame,
+		   do_writev,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_writev(frame, this);
+	return 0;	
+}
+
+static int32_t writev_trivial_completion(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 struct iatt *buf,
+					 dict_t *dict)
+{
+	crypt_local_t *local = frame->local;
+	
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->prebuf = *buf;
+	local->postbuf = *buf;
+
+	local->prebuf.ia_size = local->cur_file_size;
+	local->postbuf.ia_size = local->cur_file_size;
+
+	get_one_call(frame);
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+int crypt_writev(call_frame_t *frame,
+		 xlator_t *this,
+		 fd_t *fd,
+		 struct iovec *vec,
+		 int32_t count,
+		 off_t offset,
+		 uint32_t flags,
+		 struct iobref *iobref,
+		 dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+#if DEBUG_CRYPT
+	gf_log ("crypt", GF_LOG_DEBUG, "writing %d bytes from offset %llu",
+		(int)iovec_get_size(vec, count), (long long)offset);
+#endif
+	local = crypt_alloc_local(frame, this, GF_FOP_WRITE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	if (iobref)
+		local->iobref = iobref_ref(iobref);
+	/*
+	 * to update real file size on the server
+	 */
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->flags = flags;
+
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (iovec_get_size(vec, count) == 0)
+		goto trivial;
+
+	ret = prepare_for_submit_data(frame, this, offset,
+				      iovec_get_size(vec, count),
+				      vec, count, 0 /* don't setup gup
+						       in tail: we don't
+						       know file size yet */);
+	if (ret)
+		goto error;
+
+	if (parent_is_crypt_xlator(frame, this)) {
+		data_t *data;
+		/*
+		 * we are called by shinking crypt_ftruncate(),
+		 * which doesn't perform hole conversion;
+		 *
+		 * don't ask for access:
+		 * it has already been acquired
+		 */
+
+		/*
+		 * extract file size
+		 */
+		if (!xdata) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size hasn't been passed");
+			ret = EIO;
+			goto error;
+		}
+		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
+		if (!data) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size not found");
+			ret = EIO;
+			goto error;
+		}
+		local->old_file_size =
+			local->cur_file_size = data_to_uint64(data);
+
+		submit_data(frame, this);
+		return 0;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	/*
+	 * lock the file and retrieve its size
+	 */
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_writev_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   writev_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	if (local && local->fd)
+		fd_unref(fd);
+	if (local && local->iobref)
+		iobref_unref(iobref);
+	if (local && local->xdata)
+		dict_unref(xdata);
+	if (local && local->xattr)
+		dict_unref(local->xattr);
+	if (local && local->info)
+		free_inode_info(local->info);
+
+	STACK_UNWIND_STRICT(writev, frame, -1, ret, NULL, NULL, NULL);
+	return 0;
+}
+
+int32_t prepare_for_prune(call_frame_t *frame, xlator_t *this, uint64_t offset)
+{
+	set_config_offsets(frame, this,
+			   offset,
+			   0, /* count */
+			   DATA_ATOM,
+			   0 /* since we prune, there is no
+				gap in tail to uptodate */);
+	return 0;
+}
+
+/*
+ * Finish the read-prune-modify sequence
+ *
+ * Can be invoked as
+ * 1) ->ftruncate_cbk() for cblock-aligned, or trivial prune
+ * 2) ->writev_cbk() for non-cblock-aligned prune
+ */
+
+static int32_t prune_complete(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      struct iatt *prebuf,
+			      struct iatt *postbuf,
+			      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * This is called as ->ftruncate_cbk()
+ *
+ * Perform the "write" component of the
+ * read-prune-write sequence.
+ *
+ * submuit the rest of the file
+ */
+static int32_t prune_submit_file_tail(call_frame_t *frame,
+				      void *cookie,
+				      xlator_t *this,
+				      int32_t op_ret,
+				      int32_t op_errno,
+				      struct iatt *prebuf,
+				      struct iatt *postbuf,
+				      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	dict_t *dict;
+
+	if (op_ret < 0)
+		goto put_one_call;
+
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	dict = dict_new();
+	if (!dict) {
+		op_errno = ENOMEM;
+		goto error;
+	}
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+	local->new_file_size = conf->orig_offset;
+
+	/*
+	 * The rest of the file is a partial block and, hence,
+	 * should be written via RMW sequence, so the crypt xlator
+	 * does STACK_WIND to itself.
+	 *
+	 * Pass current file size to crypt_writev()
+	 */
+	op_errno = dict_set(dict,
+			    FSIZE_XATTR_PREFIX,
+			    data_from_uint64(local->cur_file_size));
+	if (op_errno) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "can not set key to update file size");
+		dict_unref(dict);
+		goto error;
+	}
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "passing current file size (%llu) to crypt_writev",
+	       (unsigned long long)local->cur_file_size);
+	/*
+	 * Padding will be filled with
+	 * zeros by rmw_partial_block()
+	 */
+	STACK_WIND(frame,
+		   prune_complete,
+		   this,
+		   this->fops->writev, /* crypt_writev */
+		   local->fd,
+		   &local->vec,
+		   1,
+		   conf->aligned_offset, /* offset to write from */
+		   0,
+		   local->iobref,
+		   dict);
+
+	dict_unref(dict);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * This is called as a callback of ->writev() invoked in behalf
+ * of ftruncate(): it can be
+ * 1) ordered writes issued by hole conversion in the case of
+ *    expanded truncate, or
+ * 2) an rmw partial data block issued by non-cblock-aligned
+ * prune.
+ */
+int32_t end_writeback_ftruncate(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				struct iatt *prebuf,
+				struct iatt *postbuf,
+				dict_t *xdata)
+{
+	crypt_local_t  *local = frame->local;
+	/*
+	 * if nothing has been written,
+	 * then it must be an error
+	 */
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto put_one_call;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	if (data_write_in_progress(local))
+		/* case (2) */
+		goto put_one_call;
+	/* case (1) */
+	if (should_resume_submit_hole(local))
+		submit_hole(frame, this);
+	/*
+	 * case of hole, when we should't resume
+	 */
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * Perform prune and write components of the
+ * read-prune-write sequence.
+ *
+ * Called as ->readv_cbk()
+ *
+ * Pre-conditions:
+ * @vec contains the latest atom of the file
+ * (plain text)
+ */
+static int32_t prune_write(call_frame_t *frame,
+			   void *cookie,
+			   xlator_t *this,
+			   int32_t op_ret,
+			   int32_t op_errno,
+			   struct iovec *vec,
+			   int32_t count,
+			   struct iatt *stbuf,
+			   struct iobref *iobref,
+			   dict_t *xdata)
+{
+	int32_t i;
+	size_t to_copy;
+	size_t copied = 0;
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	if (op_ret == -1)
+		goto put_one_call;
+
+	/*
+	 * At first, uptodate head block
+	 */
+	if (iovec_get_size(vec, count) < conf->off_in_head) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Failed to uptodate head block for prune");
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto put_one_call;
+	}
+	local->vec.iov_len = conf->off_in_head;
+	local->vec.iov_base = GF_CALLOC(1, local->vec.iov_len,
+					gf_crypt_mt_data);
+
+	if (local->vec.iov_base == NULL) {
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+	}
+	for (i = 0; i < count; i++) {
+		to_copy = vec[i].iov_len;
+		if (to_copy > local->vec.iov_len - copied)
+			to_copy = local->vec.iov_len - copied;
+
+		memcpy((char *)local->vec.iov_base + copied,
+		       vec[i].iov_base,
+		       to_copy);
+		copied += to_copy;
+		if (copied == local->vec.iov_len)
+			break;
+	}
+	/*
+	 * perform prune with aligned offset
+	 * (i.e. at this step we prune a bit
+	 * more then it is needed
+	 */
+	STACK_WIND(frame,
+		   prune_submit_file_tail,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->ftruncate,
+		   local->fd,
+		   conf->aligned_offset,
+		   local->xdata);
+	return 0;
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * Perform a read-prune-write sequence
+ */
+int32_t read_prune_write(call_frame_t *frame, xlator_t *this)
+{
+	int32_t ret = 0;
+	dict_t *dict = NULL;
+	crypt_local_t  *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_local_io_params_ftruncate(frame, object);
+	get_one_call_nolock(frame);
+
+	if ((conf->orig_offset & (object_alg_blksize(object) - 1)) == 0) {
+		/*
+		 * cblock-aligned prune:
+		 * we don't need read and write components,
+		 * just cut file body
+		 */
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "prune without RMW (at offset %llu",
+		       (unsigned long long)conf->orig_offset);
+
+		STACK_WIND(frame,
+			   prune_complete,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->ftruncate,
+			   local->fd,
+			   conf->orig_offset,
+			   local->xdata);
+		return 0;
+	}
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "prune with RMW (at offset %llu",
+	       (unsigned long long)conf->orig_offset);	
+	/*
+	 * We are about to perform the "read" component of the
+	 * read-prune-write sequence. It means that we need to
+	 * read encrypted data from disk and decrypt it.
+	 * So, the crypt translator does STACK_WIND to itself.
+	 * 
+	 * Pass current file size to crypt_readv()
+	 
+	 */
+	dict = dict_new();
+	if (!dict) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
+		ret = ENOMEM;
+		goto exit;
+	}
+	ret = dict_set(dict,
+		       FSIZE_XATTR_PREFIX,
+		       data_from_uint64(local->cur_file_size));
+	if (ret) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   prune_write,
+		   this,
+		   this->fops->readv, /* crypt_readv */
+		   local->fd,
+		   get_atom_size(object), /* bytes to read */
+		   conf->aligned_offset, /* offset to read from */
+		   0,
+		   dict);
+ exit:
+	if (dict)
+		dict_unref(dict);
+	return ret;
+}
+
+/*
+ * File prune is more complicated than expand.
+ * First we need to read the latest atom to not lose info
+ * needed for proper update. Also we need to make sure that
+ * every component of read-prune-write sequence leaves data
+ * consistent
+ *
+ * Non-cblock aligned prune is performed as read-prune-write
+ * sequence:
+ *
+ * 1) read the latest atom;
+ * 2) perform cblock-aligned prune
+ * 3) issue a write request for the end-of-file
+ */
+int32_t prune_file(call_frame_t *frame, xlator_t *this, uint64_t offset)
+{
+	int32_t ret;
+
+	ret = prepare_for_prune(frame, this, offset);
+	if (ret)
+		return ret;
+	return read_prune_write(frame, this);
+}
+
+int32_t expand_file(call_frame_t *frame, xlator_t *this,
+		    uint64_t offset)
+{
+	int32_t ret;
+	crypt_local_t *local = frame->local;
+
+	ret = prepare_for_submit_hole(frame, this,
+				      local->old_file_size,
+				      offset - local->old_file_size);
+	if (ret)
+		return ret;
+	submit_hole(frame, this);
+	return 0;
+}
+
+static int32_t ftruncate_trivial_completion(call_frame_t *frame,
+					    void *cookie,
+					    xlator_t *this,
+					    int32_t op_ret,
+					    int32_t op_errno,
+					    struct iatt *buf,
+					    dict_t *dict)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->prebuf = *buf;
+	local->postbuf = *buf;
+
+	local->prebuf.ia_size = local->cur_file_size;
+	local->postbuf.ia_size = local->cur_file_size;
+
+	get_one_call(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+static int32_t do_ftruncate(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    dict_t *dict,
+			    dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	if (op_ret)
+		goto error;	
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_errno = EIO;
+		goto error;
+	}
+	local->old_file_size = local->cur_file_size = data_to_uint64(data);
+
+	if (local->data_conf.orig_offset == local->cur_file_size) {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "trivial ftruncate (current file size %llu)",
+		       (unsigned long long)local->cur_file_size);
+#endif
+		goto trivial;
+	}
+	else if (local->data_conf.orig_offset < local->cur_file_size) {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG, "prune from %llu to %llu",
+		       (unsigned long long)local->cur_file_size,
+		       (unsigned long long)local->data_conf.orig_offset);
+#endif
+		op_errno = prune_file(frame,
+				      this,
+				      local->data_conf.orig_offset);
+	}
+	else {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG, "expand from %llu to %llu",
+		       (unsigned long long)local->cur_file_size,
+		       (unsigned long long)local->data_conf.orig_offset);
+#endif
+		op_errno = expand_file(frame,
+				       this,
+				       local->data_conf.orig_offset);
+	}
+	if (op_errno)
+		goto error;
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   ftruncate_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	/*
+	 * finish with ftruncate
+	 */
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+
+	get_one_call_nolock(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+static int32_t crypt_ftruncate_finodelk_cbk(call_frame_t *frame,
+					    void *cookie,
+					    xlator_t *this,
+					    int32_t op_ret,
+					    int32_t op_errno,
+					    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size first
+	 */
+	STACK_WIND(frame,
+		   do_ftruncate,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * ftruncate is performed in 2 steps:
+ * . recieve file size;
+ * . expand or prune file.
+ */
+static int32_t crypt_ftruncate(call_frame_t *frame,
+			       xlator_t *this,
+			       fd_t *fd,
+			       off_t offset,
+			       dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+
+	local = crypt_alloc_local(frame, this, GF_FOP_FTRUNCATE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		goto error;
+	}
+	local->data_conf.orig_offset = offset;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_ftruncate_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:
+	if (local && local->fd)
+		fd_unref(fd);
+	if (local && local->xdata)
+		dict_unref(xdata);
+	if (local && local->xattr)
+		dict_unref(local->xattr);
+	if (local && local->info)
+		free_inode_info(local->info);
+
+	STACK_UNWIND_STRICT(ftruncate, frame, -1, ret, NULL, NULL, NULL);
+	return 0;
+}
+
+/* ->flush_cbk() */
+int32_t truncate_end(call_frame_t *frame,
+		     void *cookie,
+		     xlator_t *this,
+		     int32_t op_ret,
+		     int32_t op_errno,
+		     dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_UNWIND_STRICT(truncate,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local->xdata);
+	return 0;
+}
+
+/* ftruncate_cbk() */
+int32_t truncate_flush(call_frame_t *frame,
+		       void *cookie,
+		       xlator_t *this,
+		       int32_t op_ret,
+		       int32_t op_errno,
+		       struct iatt *prebuf,
+		       struct iatt *postbuf,
+		       dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *fd = local->fd;
+	local->prebuf = *prebuf;
+	local->postbuf = *postbuf;
+
+	STACK_WIND(frame,
+		   truncate_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   fd,
+		   NULL);
+	fd_unref(fd);
+	return 0;
+}
+
+/*
+ * is called as ->open_cbk()
+ */
+static int32_t truncate_begin(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      fd_t *fd,
+			      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0) {
+		fd_unref(fd);
+		STACK_UNWIND_STRICT(truncate,
+				    frame,
+				    op_ret,
+				    op_errno, NULL, NULL, NULL);
+		return 0;
+	}
+	/*
+	 * crypt_truncate() is implemented via crypt_ftruncate(),
+	 * so the crypt xlator does STACK_WIND to itself here
+	 */
+	STACK_WIND(frame,
+		   truncate_flush,
+		   this,
+		   this->fops->ftruncate, /* crypt_ftruncate */
+		   fd,
+		   local->offset,
+		   NULL);
+	return 0;
+}
+
+/*
+ * crypt_truncate() is implemented via crypt_ftruncate() as a
+ * sequence crypt_open() - crypt_ftruncate() - truncate_flush()
+ */
+int32_t crypt_truncate(call_frame_t *frame,
+		       xlator_t *this,
+		       loc_t *loc,
+		       off_t offset,
+		       dict_t *xdata)
+{
+	fd_t *fd;
+	crypt_local_t *local;
+
+#if DEBUG_CRYPT
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "truncate file %s at offset %llu",
+	       loc->path, (unsigned long long)offset);
+#endif
+	local = crypt_alloc_local(frame, this, GF_FOP_TRUNCATE);
+	if (!local)
+		goto error;
+
+	fd = fd_create(loc->inode, frame->root->pid);
+	if (!fd) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
+		goto error;
+	}
+	local->fd = fd;
+	local->offset = offset;
+	local->xdata = xdata;
+	STACK_WIND(frame,
+		   truncate_begin,
+		   this,
+		   this->fops->open, /* crypt_open() */
+		   loc,
+		   O_RDWR,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(truncate, frame, -1, EINVAL, NULL, NULL, NULL);
+	return 0;
+}
+
+end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop)
+{
+	switch (fop) {
+	case GF_FOP_WRITE:
+		return end_writeback_writev;
+	case GF_FOP_FTRUNCATE:
+		return end_writeback_ftruncate;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad wb operation %d", fop);
+		return NULL;
+	}
+}
+
+/*
+ * true, if the caller needs metadata string
+ */
+static int32_t is_custom_mtd(dict_t *xdata)
+{
+	data_t *data;
+	uint32_t flags;
+
+	if (!xdata)
+		return 0;
+
+	data = dict_get(xdata, MSGFLAGS_PREFIX);
+	if (!data)
+		return 0;
+	if (data->len != sizeof(uint32_t)) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "Bad msgflags size (%d)", data->len);
+		return -1;
+	}
+	flags = *((uint32_t *)data->data);
+	return msgflags_check_mtd_lock(&flags);
+}
+
+static int32_t crypt_open_done(call_frame_t *frame,
+			       void *cookie,
+			       xlator_t *this,
+			       int32_t op_ret,
+			       int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	if (op_ret < 0)
+		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
+		       op_errno);
+	put_one_call_open(frame);
+	return 0;
+}
+
+static void crypt_open_tail(call_frame_t *frame, xlator_t *this)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   crypt_open_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+}
+
+/*
+ * load private inode info at open time
+ * called as ->fgetxattr_cbk()
+ */
+static int load_mtd_open(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *dict,
+			 dict_t *xdata)
+{
+	int32_t ret;
+	gf_boolean_t upload_info;
+	data_t *mtd;
+	uint64_t value = 0;
+	struct crypt_inode_info *info;
+	crypt_local_t *local = frame->local;
+	crypt_private_t *priv = this->private;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto exit;
+	if (op_ret < 0)
+		goto exit;
+	/*
+	 * first, check for cached info
+	 */
+	ret = inode_ctx_get(local->fd->inode, this, &value);
+	if (ret != -1) {
+		info = (struct crypt_inode_info *)(long)value;
+		if (info == NULL) {
+			gf_log(this->name, GF_LOG_WARNING,
+			       "Inode info expected, but not found");
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+		/*
+		 * info has been found in the cache
+		 */
+		upload_info = _gf_false;
+	}
+	else {
+		/*
+		 * info hasn't been found in the cache.
+		 */
+		info = alloc_inode_info(local, local->loc);
+		if (!info) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			goto exit;
+		}
+		init_inode_info_head(info, local->fd);
+		upload_info = _gf_true;
+	}
+	/*
+	 * extract metadata
+	 */
+	mtd = dict_get(dict, CRYPTO_FORMAT_PREFIX);
+	if (!mtd) {
+		local->op_ret = -1;
+		local->op_errno = ENOENT;
+		gf_log (this->name, GF_LOG_WARNING,
+			"Format string wasn't found");
+		goto exit;
+	}
+	/*
+	 * authenticate metadata against the path
+	 */	
+	ret = open_format((unsigned char *)mtd->data,
+			  mtd->len,
+			  local->loc,
+			  info,
+			  get_master_cinfo(priv),
+			  local,
+			  upload_info);
+	if (ret) {
+		local->op_ret = -1;
+		local->op_errno = ret;
+		goto exit;
+	}
+	if (upload_info) {
+		ret = init_inode_info_tail(info, get_master_cinfo(priv));
+		if (ret) {
+			local->op_ret = -1;
+			local->op_errno = ret;
+			goto exit;	
+		}
+		ret = inode_ctx_put(local->fd->inode,
+				    this, (uint64_t)(long)info);
+		if (ret == -1) {
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+	}
+	if (local->custom_mtd) {
+		/*
+		 * pass the metadata string to the customer
+		 */
+		ret = dict_set_static_bin(local->xdata,
+					  CRYPTO_FORMAT_PREFIX,
+					  mtd->data,
+					  mtd->len);
+		if (ret) {
+			local->op_ret = -1;
+			local->op_errno = ret;
+			goto exit;
+		}
+	}
+ exit:
+	if (!local->custom_mtd)
+		crypt_open_tail(frame, this);
+	else
+		put_one_call_open(frame);
+	return 0;
+}
+
+static int32_t crypt_open_finodelk_cbk(call_frame_t *frame,
+				       void *cookie,
+				       xlator_t *this,
+				       int32_t op_ret,
+				       int32_t op_errno,
+				       dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING, "finodelk (LOCK) failed");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   load_mtd_open,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   CRYPTO_FORMAT_PREFIX,
+		   NULL);
+	return 0;
+ exit:
+	put_one_call_open(frame);
+	return 0;
+}
+
+/*
+ * verify metadata against the specified pathname
+ */
+static int32_t crypt_open_cbk(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      fd_t *fd,
+			      dict_t *xdata)
+{
+	struct gf_flock lock = {0, };
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto exit;
+	if (op_ret < 0)
+		goto exit;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	else if (local->custom_mtd){
+		local->xdata = dict_new();
+		if (!local->xdata) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			gf_log ("crypt", GF_LOG_ERROR,
+				"Can not get new dict for mtd string");
+			goto exit;
+		}
+	}
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = local->custom_mtd ? F_WRLCK : F_RDLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_open_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ exit:
+	put_one_call_open(frame);
+	return 0;
+}
+
+static int32_t crypt_open(call_frame_t *frame,
+			  xlator_t *this,
+			  loc_t *loc,
+			  int32_t flags,
+			  fd_t *fd,
+			  dict_t *xdata)
+{
+	int32_t ret = ENOMEM;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_OPEN);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc) {
+		ret = ENOMEM;
+		goto error;
+	}
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	ret = is_custom_mtd(xdata);
+	if (ret < 0) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+		ret = EINVAL;
+		goto error;
+	}
+	local->custom_mtd = ret;
+
+	if ((flags & O_ACCMODE) == O_WRONLY)
+		/*
+		 * we can't open O_WRONLY, because
+		 * we need to do read-modify-write
+		 */
+		flags = (flags & ~O_ACCMODE) | O_RDWR;
+	/*
+	 * Make sure that out translated offsets
+	 * and counts won't be ignored
+	 */
+	flags &= ~O_APPEND;
+	get_one_call_nolock(frame);
+	STACK_WIND(frame,
+		   crypt_open_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->open,
+		   loc,
+		   flags,
+		   fd,
+		   xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(open,
+			    frame,
+			    -1,
+			    ret,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t init_inode_info_tail(struct crypt_inode_info *info,
+				    struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct object_cipher_info *object = &info->cinfo;
+
+#if DEBUG_CRYPT
+	gf_log("crypt", GF_LOG_DEBUG, "Init inode info for object %s",
+	       uuid_utoa(info->oid));
+#endif
+	ret = data_cipher_algs[object->o_alg][object->o_mode].set_private(info,
+									master);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set private info failed");
+		return ret;
+	}
+	return 0;
+}
+
+/*
+ * Init inode info at ->create() time
+ */
+static void init_inode_info_create(struct crypt_inode_info *info,
+				   struct master_cipher_info *master,
+				   data_t *data)
+{
+	struct object_cipher_info *object;
+
+	info->nr_minor = CRYPT_XLATOR_ID;
+	memcpy(info->oid, data->data, data->len);
+
+	object = &info->cinfo;
+
+	object->o_alg        = master->m_alg;
+	object->o_mode       = master->m_mode;
+	object->o_block_bits = master->m_block_bits;
+	object->o_dkey_size  = master->m_dkey_size;
+}
+
+static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd)
+{
+	memcpy(info->oid, fd->inode->gfid, sizeof(uuid_t));
+}
+
+static int32_t crypt_create_done(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno, dict_t *xdata)
+{
+	crypt_private_t *priv = this->private;
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	if (op_ret < 0) {
+		free_inode_info(info);
+		goto unwind;
+	}
+	op_errno = init_inode_info_tail(info, get_master_cinfo(priv));
+	if (op_errno) {
+		op_ret = -1;
+		free_inode_info(info);
+		goto unwind;
+	}
+	/*
+	 * FIXME: drop major subversion number
+	 */
+	op_ret = inode_ctx_put(local->fd->inode, this, (uint64_t)(long)info);
+	if (op_ret == -1) {
+		op_errno = EIO;
+		free_inode_info(info);
+		goto unwind;
+	}
+ unwind:
+	free_format(local);
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    local_fd,
+			    local_inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	inode_unref(local_inode);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int crypt_create_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     dict_t *xdata)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	dict_unref(local->xattr);
+
+	if (op_ret < 0)
+		goto error;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   crypt_create_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:
+	free_inode_info(local->info);
+	free_format(local);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    local_fd,
+			    local_inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+
+	fd_unref(local_fd);
+	inode_unref(local_inode);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int32_t crypt_create_finodelk_cbk(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+
+	if (op_ret < 0)
+		goto error;
+
+	STACK_WIND(frame,
+		   crypt_create_tail,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fsetxattr,
+		   local->fd,
+		   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+		   0,
+		   NULL);
+	return 0;
+ error: 
+	free_inode_info(info);
+	free_format(local);
+	fd_unref(local->fd);
+	dict_unref(local->xattr);
+	if (local->xdata)
+		dict_unref(local->xdata);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+/*
+ * Create and store crypt-specific format on disk;
+ * Populate cache with private inode info
+ */
+static int32_t crypt_create_cbk(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				fd_t *fd,
+				inode_t *inode,
+				struct iatt *buf,
+				struct iatt *preparent,
+				struct iatt *postparent,
+				dict_t *xdata)
+{
+	struct gf_flock lock = {0, };
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+
+	if (op_ret < 0)
+		goto error;
+	if (xdata)
+		local->xdata = dict_ref(xdata);	
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+
+        lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+        STACK_WIND(frame,
+		   crypt_create_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);	
+	return 0;
+ error:
+	free_inode_info(info);
+	free_format(local);
+	fd_unref(local->fd);
+	dict_unref(local->xattr);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    NULL, NULL, NULL,
+			    NULL, NULL, NULL);
+	return 0;
+}
+
+static int32_t crypt_create(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc,
+			    int32_t flags,
+			    mode_t mode,
+			    mode_t umask,
+			    fd_t *fd,
+			    dict_t *xdata)
+{
+	int ret;
+	data_t *data;
+	crypt_local_t *local;
+	crypt_private_t *priv;
+	struct master_cipher_info *master;
+	struct crypt_inode_info *info;
+
+	priv = this->private;
+	master = get_master_cinfo(priv);
+
+	if (master_alg_atomic(master)) {
+		/*
+		 * We can't open O_WRONLY, because we
+		 * need to do read-modify-write.
+		 */
+		if ((flags & O_ACCMODE) == O_WRONLY)
+			flags = (flags & ~O_ACCMODE) | O_RDWR;
+		/*
+		 * Make sure that out translated offsets
+		 * and counts won't be ignored
+		 */
+		flags &= ~O_APPEND;
+	}
+	local = crypt_alloc_local(frame, this, GF_FOP_CREATE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	data = dict_get(xdata, "gfid-req");
+	if (!data) {
+		ret = EINVAL;
+		gf_log("crypt", GF_LOG_WARNING, "gfid not found");
+		goto error;
+	}
+	if (data->len != sizeof(uuid_t)) {
+		ret = EINVAL;
+		gf_log("crypt", GF_LOG_WARNING,
+		       "bad gfid size (%d), should be %d",
+		       (int)data->len, (int)sizeof(uuid_t));
+		goto error;
+	}
+	info = alloc_inode_info(local, loc);
+	if (!info){
+		ret = ENOMEM;
+		goto error;
+	}
+	/*
+	 * NOTE:
+	 * format has to be created BEFORE
+	 * proceeding to the untrusted server
+	 */
+	ret = alloc_format_create(local);
+	if (ret) {
+		free_inode_info(info);
+		goto error;
+	}
+	init_inode_info_create(info, master, data);
+
+	ret = create_format(local->format,
+			    loc,
+			    info,
+			    master);
+	if (ret) {
+		free_inode_info(info);
+		goto error;		
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		free_inode_info(info);
+		free_format(local);	
+		goto error;
+	}
+	ret = dict_set_static_bin(local->xattr,
+				  CRYPTO_FORMAT_PREFIX,
+				  local->format,
+				  new_format_size());
+	if (ret) {
+		dict_unref(local->xattr);
+		free_inode_info(info);
+		free_format(local);
+		goto error;
+	}
+	ret = dict_set(local->xattr, FSIZE_XATTR_PREFIX, data_from_uint64(0));
+	if (ret) {
+		dict_unref(local->xattr);
+		free_inode_info(info);
+		free_format(local);
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	STACK_WIND(frame,
+		   crypt_create_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->create,
+		   loc,
+		   flags,
+		   mode,
+		   umask,
+		   fd,
+		   xdata);
+	return 0;
+ error:
+	gf_log("crypt", GF_LOG_WARNING, "can not create file");
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    -1,
+			    ret,
+			    NULL, NULL, NULL,
+			    NULL, NULL, NULL);
+	return 0;
+}
+
+/*
+ * FIXME: this should depends on the version of format string
+ */
+static int32_t filter_crypt_xattr(dict_t *dict,
+				  char *key, data_t *value, void *data)
+{
+        dict_del(dict, key);
+        return 0;
+}
+
+static int32_t crypt_fsetxattr(call_frame_t *frame,
+			       xlator_t *this,
+			       fd_t *fd,
+			       dict_t *dict,
+			       int32_t flags, dict_t *xdata)
+{
+	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
+			     filter_crypt_xattr, NULL);
+	STACK_WIND(frame,
+		   default_fsetxattr_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fsetxattr,
+		   fd,
+		   dict,
+		   flags,
+		   xdata);
+	return 0;
+}
+
+/*
+ * TBD: verify file metadata before wind
+ */
+static int32_t crypt_setxattr(call_frame_t *frame,
+			      xlator_t *this,
+			      loc_t *loc,
+			      dict_t *dict,
+			      int32_t flags, dict_t *xdata)
+{
+	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
+			     filter_crypt_xattr, NULL);
+	STACK_WIND(frame,
+		   default_setxattr_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->setxattr,
+		   loc,
+		   dict,
+		   flags,
+		   xdata);
+	return 0;
+}
+
+/*
+ * called as flush_cbk()
+ */
+static int32_t linkop_end(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	linkop_unwind_handler_t unwind_fn;
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0 &&
+	    op_errno == ENOENT &&
+	    local->loc->inode->ia_type == IA_IFLNK) {
+		local->op_ret = 0;
+		local->op_errno = 0;
+	}
+	unwind_fn(frame);
+	return 0;
+}
+
+/*
+ * unpin inode on the server
+ */
+static int32_t link_flush(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  inode_t *inode,
+			  struct iatt *buf,
+			  struct iatt *preparent,
+			  struct iatt *postparent, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+
+	STACK_WIND(frame,
+		   linkop_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	link_unwind(frame);
+	return 0;
+}
+
+void link_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+	inode_t *inode;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(link,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL);
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	inode = local->inode;
+
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->newloc) {
+		loc_wipe(local->newloc);
+		GF_FREE(local->newloc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(link,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+	if (inode)
+		inode_unref(inode);
+}
+
+void link_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   link_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->link,
+		   local->loc,
+		   local->newloc,
+		   local->xdata);
+}
+
+/*
+ * unlink()
+ */
+static int32_t unlink_flush(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    struct iatt *preparent,
+			    struct iatt *postparent, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	STACK_WIND(frame,
+		   linkop_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	unlink_unwind(frame);
+	return 0;
+}
+
+void unlink_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(unlink,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL);
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(unlink,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+}
+
+void unlink_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   unlink_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->unlink,
+		   local->loc,
+		   local->flags,
+		   local->xdata);
+}
+
+void rename_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+	struct iatt *prenewparent;
+	struct iatt *postnewparent;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(rename,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL);	
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	prenewparent = local->prenewparent;
+	postnewparent = local->postnewparent;	
+
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->newloc){
+		loc_wipe(local->newloc);
+		GF_FREE(local->newloc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(rename,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    prenewparent,
+			    postnewparent,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+	if (prenewparent)
+		GF_FREE(prenewparent);
+	if (postnewparent)
+		GF_FREE(postnewparent);
+}
+
+/*
+ * called as flush_cbk()
+ */
+static int32_t rename_end(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	rename_unwind(frame);
+	return 0;
+}
+
+static int32_t rename_flush(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    struct iatt *buf,
+			    struct iatt *preoldparent,
+			    struct iatt *postoldparent,
+			    struct iatt *prenewparent,
+			    struct iatt *postnewparent,
+			    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	dict_unref(local->xdata);
+	local->xdata = NULL;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	local->buf = *buf;
+	local->prebuf = *preoldparent;
+	local->postbuf = *postoldparent;
+	if (prenewparent) {
+		local->prenewparent = GF_CALLOC(1, sizeof(*prenewparent),
+						gf_crypt_mt_iatt);
+		if (!local->prenewparent) {
+			op_errno = ENOMEM;
+			goto error;
+		}
+		*local->prenewparent = *prenewparent;
+	}
+	if (postnewparent) {
+		local->postnewparent = GF_CALLOC(1, sizeof(*postnewparent),
+						 gf_crypt_mt_iatt);
+		if (!local->postnewparent) {
+			op_errno = ENOMEM;
+			goto error;
+		}
+		*local->postnewparent = *postnewparent;
+	}
+	STACK_WIND(frame,
+		   rename_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	rename_unwind(frame);
+	return 0;
+}
+
+void rename_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   rename_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->rename,
+		   local->loc,
+		   local->newloc,
+		   local->xdata);
+}
+
+static int32_t __do_linkop(call_frame_t *frame,
+			   void *cookie,
+			   xlator_t *this,
+			   int32_t op_ret,
+			   int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	linkop_wind_handler_t wind_fn;
+	linkop_unwind_handler_t unwind_fn;
+
+	wind_fn = linkop_wind_dispatch(local->fop);
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret >= 0)
+		wind_fn(frame, this);
+	else {
+		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
+		       op_errno);
+		unwind_fn(frame);
+	}
+	return 0;
+}
+
+static int32_t do_linkop(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *xdata)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+	linkop_unwind_handler_t unwind_fn;
+
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if(op_ret < 0)
+		goto error;
+
+	lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   __do_linkop,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:		
+	unwind_fn(frame);
+	return 0;
+}
+
+/*
+ * Update the metadata string (against the new pathname);
+ * submit the result
+ */
+static int32_t linkop_begin(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    fd_t *fd,
+			    dict_t *xdata)
+{
+	gf_boolean_t upload_info;
+	crypt_local_t *local = frame->local;
+	crypt_private_t *priv = this->private;
+	struct crypt_inode_info *info;
+	data_t *old_mtd;
+	uint32_t new_mtd_size;
+	uint64_t value = 0;
+	void (*unwind_fn)(call_frame_t *frame);
+	void (*wind_fn)(call_frame_t *frame, xlator_t *this);
+	mtd_op_t mop;
+
+	wind_fn = linkop_wind_dispatch(local->fop);
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+	mop = linkop_mtdop_dispatch(local->fop);
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto wind;
+	if (op_ret < 0)
+		/*
+		 * verification failed
+		 */
+		goto error;
+
+	old_mtd = dict_get(xdata, CRYPTO_FORMAT_PREFIX);
+	if (!old_mtd) {
+		op_errno = EIO;
+		gf_log (this->name, GF_LOG_DEBUG,
+			"Metadata string wasn't found");
+		goto error;
+	}
+	new_mtd_size = format_size(mop, old_mtd->len);
+	op_errno = alloc_format(local, new_mtd_size);
+	if (op_errno)
+		goto error;
+	/*
+	 * check for cached info 
+	 */
+	op_ret = inode_ctx_get(fd->inode, this, &value);
+	if (op_ret != -1) {
+		info = (struct crypt_inode_info *)(long)value;
+		if (info == NULL) {
+			gf_log (this->name, GF_LOG_WARNING,
+				"Inode info was not found");
+			op_errno = EINVAL;
+			goto error;
+		}
+		/*
+		 * info was found in the cache
+		 */
+		local->info = info;
+		upload_info = _gf_false;
+	}
+	else {
+		/*
+		 * info wasn't found in the cache;
+		 */
+		info = alloc_inode_info(local, local->loc);
+		if (!info)
+			goto error;
+		init_inode_info_head(info, fd);
+		local->info = info;
+		upload_info = _gf_true;
+	}
+	op_errno = open_format((unsigned char *)old_mtd->data,
+			       old_mtd->len,
+			       local->loc,
+			       info,
+			       get_master_cinfo(priv),
+			       local,
+			       upload_info);
+	if (op_errno)
+		goto error;
+	if (upload_info == _gf_true) {
+		op_errno = init_inode_info_tail(info,
+						get_master_cinfo(priv));
+		if (op_errno)
+			goto error;
+		op_errno = inode_ctx_put(fd->inode, this,
+					 (uint64_t)(long)(info));
+		if (op_errno == -1) {
+			op_errno = EIO;
+			goto error;
+		}
+	}
+	/*
+	 * update the format string (append/update/cup a MAC)
+	 */
+	op_errno = update_format(local->format,
+				 (unsigned char *)old_mtd->data,
+				 old_mtd->len,
+				 local->mac_idx,
+				 mop,
+				 local->newloc,
+				 info,
+				 get_master_cinfo(priv),
+				 local);
+	if (op_errno)
+		goto error;
+	/*
+	 * store the new format string on the server
+	 */
+	if (new_mtd_size) {
+		op_errno = dict_set_static_bin(local->xattr,
+					       CRYPTO_FORMAT_PREFIX,
+					       local->format,
+					       new_mtd_size);
+		if (op_errno)
+			goto error;
+	}
+	STACK_WIND(frame,
+		   do_linkop,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->setxattr,
+		   local->loc,
+		   local->xattr,
+		   0,
+		   NULL);
+	return 0;
+ wind:
+	wind_fn(frame, this);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	unwind_fn(frame);
+	return 0;
+}
+
+static int32_t linkop_grab_local(call_frame_t *frame,
+				 xlator_t *this,
+				 loc_t *oldloc,
+				 loc_t *newloc,
+				 int flags, dict_t *xdata,
+				 glusterfs_fop_t op)
+{
+	int32_t ret = ENOMEM;
+	fd_t *fd;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, op);
+	if (!local)
+		goto error;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	fd = fd_create(oldloc->inode, frame->root->pid);
+	if (!fd) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
+		goto error;
+	}
+	local->fd = fd;
+	local->flags = flags;
+	local->loc = GF_CALLOC(1, sizeof(*oldloc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, oldloc);
+	if (ret) {
+		GF_FREE(local->loc);
+		local->loc = NULL;
+		goto error;
+	}
+	if (newloc) {
+		local->newloc = GF_CALLOC(1, sizeof(*newloc), gf_crypt_mt_loc);
+		if (!local->newloc) {
+			GF_FREE(local->loc);
+			loc_wipe(local->loc);
+			goto error;
+		}
+		memset(local->newloc, 0, sizeof(*local->newloc));
+		ret = loc_copy(local->newloc, newloc);
+		if (ret) {
+			GF_FREE(local->loc);
+			loc_wipe(local->loc);
+			GF_FREE(local->newloc);
+			goto error;
+		}
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
+		ret = ENOMEM;
+		goto error;
+	}
+	return 0;
+ error:
+	if (local->xdata)
+		dict_unref(local->xdata);
+	if (local->fd)
+		fd_unref(local->fd);
+	local->fd = 0;
+	local->loc = NULL;
+	local->newloc = NULL;
+
+	local->op_ret = -1;
+	local->op_errno = ret;
+
+	return ret;
+}
+
+/*
+ * read and verify locked metadata against the old pathname (via open);
+ * update the metadata string in accordance with the new pathname;
+ * submit modified metadata;
+ * wind;
+ */
+static int32_t linkop(call_frame_t *frame,
+		      xlator_t *this,
+		      loc_t *oldloc,
+		      loc_t *newloc,
+		      int flags,
+		      dict_t *xdata,
+		      glusterfs_fop_t op)
+{
+	int32_t ret;
+	dict_t *dict;
+	crypt_local_t *local;
+	void (*unwind_fn)(call_frame_t *frame);
+
+	unwind_fn = linkop_unwind_dispatch(op);
+
+	ret = linkop_grab_local(frame, this, oldloc, newloc, flags, xdata, op);
+	local = frame->local;
+	if (ret)
+		goto error;
+	dict = dict_new();
+	if (!dict) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
+		ret = ENOMEM;
+		goto error;
+	}
+	/*
+	 * Set a message to crypt_open() that we need
+	 * locked metadata string.
+	 * All link operations (link, unlink, rename)
+	 * need write lock
+	 */
+	msgflags_set_mtd_wlock(&local->msgflags);
+	ret = dict_set_static_bin(dict,
+				  MSGFLAGS_PREFIX,
+				  &local->msgflags,
+				  sizeof(local->msgflags));
+	if (ret) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not set dict");
+		dict_unref(dict);
+		goto error;
+	}
+	/*
+	 * verify metadata against the old pathname
+	 * and retrieve locked metadata string
+	 */
+	STACK_WIND(frame,
+		   linkop_begin,
+		   this,
+		   this->fops->open, /* crypt_open() */
+		   oldloc,
+		   O_RDWR,
+		   local->fd,
+		   dict);
+	dict_unref(dict);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = ret;
+	unwind_fn(frame);
+	return 0;
+}
+
+static int32_t crypt_link(call_frame_t *frame, xlator_t *this,
+			  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_LINK);
+}
+
+static int32_t crypt_unlink(call_frame_t *frame, xlator_t *this,
+			    loc_t *loc, int flags, dict_t *xdata)
+{
+	return linkop(frame, this, loc, NULL, flags, xdata, GF_FOP_UNLINK);
+}
+
+static int32_t crypt_rename(call_frame_t *frame, xlator_t *this,
+			    loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_RENAME);
+}
+
+static void put_one_call_open(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		fd_t *fd = local->fd;
+		loc_t *loc = local->loc;
+		dict_t *xdata = local->xdata;
+
+		STACK_UNWIND_STRICT(open,
+				    frame,
+				    local->op_ret,
+				    local->op_errno,
+				    fd,
+				    xdata);
+		fd_unref(fd);
+		if (xdata)
+			dict_unref(xdata);
+		loc_wipe(loc);
+		GF_FREE(loc);
+	}
+}
+
+static int32_t __crypt_readv_done(call_frame_t *frame,
+				  void *cookie,
+				  xlator_t *this,
+				  int32_t op_ret,
+				  int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	/* read deals with data configs only */
+	struct iovec *avec = local->data_conf.avec;
+	char **pool = local->data_conf.pool;
+	int blocks_in_pool = local->data_conf.blocks_in_pool;
+	struct iobref *iobref = local->iobref;
+	struct iobref *iobref_data = local->iobref_data;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "readv unlock failed (%d)", op_errno);
+		if (local->op_ret >= 0) {
+			local->op_ret = op_ret;
+			local->op_errno = op_errno;
+		}
+	}
+	dump_plain_text(local, avec);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "readv: ret_to_user: %d, iovec len: %d, ia_size: %llu",
+	       (int)(local->rw_count > 0 ? local->rw_count : local->op_ret),
+	       (int)(local->rw_count > 0 ? iovec_get_size(avec, local->data_conf.acount) : 0),
+	       (unsigned long long)local->buf.ia_size);
+
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    local->rw_count > 0 ? local->rw_count : local->op_ret,
+			    local->op_errno,
+			    avec,
+			    avec ? local->data_conf.acount : 0,
+			    &local->buf,
+			    local->iobref,
+			    local_xdata);
+
+	free_avec(avec, pool, blocks_in_pool);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (iobref)
+		iobref_unref(iobref);
+	if (iobref_data)
+		iobref_unref(iobref_data);
+	return 0;
+}
+
+static void crypt_readv_done(call_frame_t *frame, xlator_t *this)
+{
+	if (parent_is_crypt_xlator(frame, this))
+		/*
+		 *  don't unlock (it will be done by the parent)
+		 */
+		__crypt_readv_done(frame, NULL, this, 0, 0, NULL);
+	else {
+		crypt_local_t *local = frame->local;
+		struct gf_flock  lock  = {0, };
+
+		lock.l_type   = F_UNLCK;
+		lock.l_whence = SEEK_SET;
+		lock.l_start  = 0;
+		lock.l_len    = 0;
+		lock.l_pid    = 0;
+
+		STACK_WIND(frame,
+			   __crypt_readv_done,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->finodelk,
+			   this->name,
+			   local->fd,
+			   F_SETLKW,
+			   &lock,
+			   NULL);
+	}
+}
+
+static void put_one_call_readv(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local))
+		crypt_readv_done(frame, this);
+}
+
+static int32_t __crypt_writev_done(call_frame_t *frame,
+				   void *cookie,
+				   xlator_t *this,
+				   int32_t op_ret,
+				   int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	int32_t ret_to_user;
+
+	if (local->xattr)
+		dict_unref(local->xattr);
+	/*
+	 * Calculate amout of butes to be returned
+	 * to user. We need to subtract paddings that
+	 * have been written as a part of atom.
+	 */
+	/*
+	 * subtract head padding
+	 */
+	if (local->rw_count == 0)
+		/*
+		 * Nothing has been written, it must be an error
+		 */
+		ret_to_user = local->op_ret;
+	else if (local->rw_count <= local->data_conf.off_in_head) {
+		gf_log("crypt", GF_LOG_WARNING, "Incomplete write");
+		ret_to_user = 0;
+	}
+	else
+		ret_to_user = local->rw_count -
+			local->data_conf.off_in_head;
+	/*
+	 * subtract tail padding
+	 */
+	if (ret_to_user > local->data_conf.orig_size)
+		ret_to_user = local->data_conf.orig_size;
+
+	if (local->iobref)
+		iobref_unref(local->iobref);
+	if (local->iobref_data)
+		iobref_unref(local->iobref_data);
+	free_avec_data(local);
+	free_avec_hole(local);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "writev: ret_to_user: %d", ret_to_user);
+
+	STACK_UNWIND_STRICT(writev,
+			    frame,
+			    ret_to_user,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int32_t crypt_writev_done(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno,
+				 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
+
+	if (parent_is_crypt_xlator(frame, this))
+		/*
+		 * don't unlock (it will be done by the parent)
+		 */
+		__crypt_writev_done(frame, NULL, this, 0, 0, NULL);
+	else {
+		struct gf_flock  lock  = {0, };
+
+		lock.l_type   = F_UNLCK;
+		lock.l_whence = SEEK_SET;
+		lock.l_start  = 0;
+		lock.l_len    = 0;
+		lock.l_pid    = 0;
+
+		STACK_WIND(frame,
+			   __crypt_writev_done,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->finodelk,
+			   this->name,
+			   local->fd,
+			   F_SETLKW,
+			   &lock,
+			   NULL);
+	}
+	return 0;
+}
+
+static void put_one_call_writev(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		if (local->update_disk_file_size) {
+			int32_t ret;
+			/*
+			 * update file size, unlock the file and unwind
+			 */
+			ret = dict_set(local->xattr,
+				       FSIZE_XATTR_PREFIX,
+				       data_from_uint64(local->cur_file_size));
+			if (ret) {
+				gf_log("crypt", GF_LOG_WARNING,
+				       "can not set key to update file size");
+				crypt_writev_done(frame, NULL,
+						  this, 0, 0, NULL);
+				return;
+			}
+			gf_log("crypt", GF_LOG_DEBUG,
+			       "Updating disk file size to %llu",
+			       (unsigned long long)local->cur_file_size);
+			STACK_WIND(frame,
+				   crypt_writev_done,
+				   FIRST_CHILD(this),
+				   FIRST_CHILD(this)->fops->fsetxattr,
+				   local->fd,
+				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+				   0,
+				   NULL);
+		}
+		else
+			crypt_writev_done(frame, NULL, this, 0, 0, NULL);
+	}
+}
+
+static int32_t __crypt_ftruncate_done(call_frame_t *frame,
+				      void *cookie,
+				      xlator_t *this,
+				      int32_t op_ret,
+				      int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	char *iobase = local->vec.iov_base;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "ftruncate unlock failed (%d)", op_errno);
+		if (local->op_ret >= 0) {
+			local->op_ret = op_ret;
+			local->op_errno = op_errno;
+		}
+	}
+	if (local->iobref_data)
+		iobref_unref(local->iobref_data);
+	free_avec_data(local);
+	free_avec_hole(local);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "ftruncate, return to user: presize=%llu, postsize=%llu",
+	       (unsigned long long)local->prebuf.ia_size,
+	       (unsigned long long)local->postbuf.ia_size);
+
+	STACK_UNWIND_STRICT(ftruncate,
+			    frame,
+			    local->op_ret < 0 ? -1 : 0,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (iobase)
+		GF_FREE(iobase);
+	return 0;
+}
+
+static int32_t crypt_ftruncate_done(call_frame_t *frame,
+				    void *cookie,
+				    xlator_t *this,
+				    int32_t op_ret,
+				    int32_t op_errno,
+				    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct gf_flock  lock  = {0, };
+
+	dict_unref(local->xattr);
+	if (op_ret < 0)
+		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
+
+	lock.l_type   = F_UNLCK;
+	lock.l_whence = SEEK_SET;
+	lock.l_start  = 0;
+	lock.l_len    = 0;
+	lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   __crypt_ftruncate_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+}
+
+static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		if (local->update_disk_file_size) {
+			int32_t ret;
+			/*
+			 * update file size, unlock the file and unwind
+			 */
+			ret = dict_set(local->xattr,
+				       FSIZE_XATTR_PREFIX,
+				       data_from_uint64(local->cur_file_size));
+			if (ret) {
+				gf_log("crypt", GF_LOG_WARNING,
+				       "can not set key to update file size");
+				crypt_ftruncate_done(frame, NULL,
+						     this, 0, 0, NULL);
+				return;
+			}
+			gf_log("crypt", GF_LOG_DEBUG,
+			       "Updating disk file size to %llu",
+			       (unsigned long long)local->cur_file_size);
+			STACK_WIND(frame,
+				   crypt_ftruncate_done,
+				   FIRST_CHILD(this),
+				   FIRST_CHILD(this)->fops->fsetxattr,
+				   local->fd,
+				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+				   0,
+				   NULL);
+		}
+		else
+			crypt_ftruncate_done(frame, NULL, this, 0, 0, NULL);
+	}
+}
+
+/*
+ * load regular file size for some FOPs
+ */
+static int32_t load_file_size(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      dict_t *dict,
+			      dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	if (op_ret < 0)
+		goto unwind;
+	/*
+	 * load regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		if (local->xdata)
+			dict_unref(local->xdata);
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_ret = -1;
+		op_errno = EIO;
+		goto unwind;
+	}
+	local->buf.ia_size = data_to_uint64(data);
+
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "FOP %d: Translate regular file to %llu",
+	       local->fop,
+	       (unsigned long long)local->buf.ia_size);
+ unwind:	
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->loc) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_UNWIND_STRICT(fstat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    local->xdata);
+		break;
+	case GF_FOP_STAT:
+		STACK_UNWIND_STRICT(stat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    local->xdata);
+		break;
+	case GF_FOP_LOOKUP:
+		STACK_UNWIND_STRICT(lookup,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? local->inode : NULL,
+				    op_ret >= 0 ? &local->buf : NULL,				    
+				    local->xdata,
+				    op_ret >= 0 ? &local->postbuf : NULL);
+		break;
+	case GF_FOP_READ:
+		STACK_UNWIND_STRICT(readv,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    NULL,
+				    0,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    NULL,
+				    NULL);
+		break;
+	default:
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Improper file operation %d", local->fop);
+	}
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (local_inode)
+		inode_unref(local_inode);
+	return 0;
+}
+
+static int32_t crypt_stat_common_cbk(call_frame_t *frame,
+				     void *cookie,
+				     xlator_t *this,
+				     int32_t op_ret,
+				     int32_t op_errno,
+				     struct iatt *buf, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto unwind;
+	if (!IA_ISREG(buf->ia_type))
+		goto unwind;
+
+	local->buf = *buf;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_WIND(frame,
+			   load_file_size,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->fgetxattr,
+			   local->fd,
+			   FSIZE_XATTR_PREFIX,
+			   NULL);
+		break;
+	case GF_FOP_STAT:
+		STACK_WIND(frame,
+			   load_file_size,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->getxattr,
+			   local->loc,
+			   FSIZE_XATTR_PREFIX,
+			   NULL);
+		break;
+	default:
+		gf_log (this->name, GF_LOG_WARNING,
+			"Improper file operation %d", local->fop);
+	}
+	return 0;
+ unwind:
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->loc) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_UNWIND_STRICT(fstat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? buf : NULL,
+				    op_ret >= 0 ? xdata : NULL);
+		break;
+	case GF_FOP_STAT:
+		STACK_UNWIND_STRICT(stat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? buf : NULL,
+				    op_ret >= 0 ? xdata : NULL);
+		break;
+	default:
+		gf_log (this->name, GF_LOG_WARNING,
+			"Improper file operation %d", local->fop);
+	}
+	return 0;
+}
+
+static int32_t crypt_fstat(call_frame_t *frame,
+			   xlator_t *this,
+			   fd_t *fd, dict_t *xdata)
+{
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_FSTAT);
+	if (!local)
+		goto error;
+	local->fd = fd_ref(fd);
+	STACK_WIND(frame,
+		   crypt_stat_common_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   xdata);	
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(fstat,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t crypt_stat(call_frame_t *frame,
+			  xlator_t *this,
+			  loc_t *loc, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_STAT);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	STACK_WIND(frame,
+		   crypt_stat_common_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->stat,
+		   loc,
+		   xdata);	
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(stat,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t crypt_lookup_cbk(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				inode_t *inode,
+				struct iatt *buf, dict_t *xdata,
+				struct iatt *postparent)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto unwind;
+	if (!IA_ISREG(buf->ia_type))
+		goto unwind;
+
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->postbuf = *postparent;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	uuid_copy(local->loc->gfid, buf->ia_gfid);
+
+	STACK_WIND(frame,
+		   load_file_size,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->getxattr,
+		   local->loc,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ unwind:
+	loc_wipe(local->loc);
+	GF_FREE(local->loc);
+	STACK_UNWIND_STRICT(lookup,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    inode,
+			    buf,
+			    xdata,
+			    postparent);
+	return 0;
+}
+
+static int32_t crypt_lookup(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_LOOKUP);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	gf_log(this->name, GF_LOG_DEBUG, "Lookup %s", loc->path);
+	STACK_WIND(frame,
+		   crypt_lookup_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->lookup,
+		   loc,
+		   xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(lookup,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+/*
+ * for every regular directory entry find its real file size
+ * and update stat's buf properly
+ */
+static int32_t crypt_readdirp_cbk(call_frame_t *frame,
+				  void *cookie,
+				  xlator_t *this,
+				  int32_t op_ret,
+				  int32_t op_errno,
+				  gf_dirent_t *entries, dict_t *xdata)
+{
+	gf_dirent_t *entry = NULL;
+
+	if (op_ret < 0)
+		goto unwind;
+
+	list_for_each_entry (entry, (&entries->list), list) {
+		data_t *data;
+
+		if (!IA_ISREG(entry->d_stat.ia_type))
+			continue;
+		data = dict_get(entry->dict, FSIZE_XATTR_PREFIX);
+		if (!data){
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size of direntry not found");
+			op_errno = EIO;
+			op_ret = -1;
+			break;
+		}
+		entry->d_stat.ia_size = data_to_uint64(data);
+	}
+ unwind:
+	STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
+	return 0;
+}
+
+/*
+ * ->readdirp() fills in-core inodes, so we need to set proper
+ * file sizes for all directory entries of the parent @fd.
+ * Actual updates take place in ->crypt_readdirp_cbk()
+ */
+static int32_t crypt_readdirp(call_frame_t *frame, xlator_t *this,
+			      fd_t *fd, size_t size, off_t offset,
+			      dict_t *xdata)
+{
+	int32_t ret = ENOMEM;
+
+	if (!xdata) {
+		xdata = dict_new();
+		if (!xdata)
+			goto error;
+	}
+	else
+		dict_ref(xdata);
+	/*
+	 * make sure that we'll have real file sizes at ->readdirp_cbk()
+	 */
+	ret = dict_set(xdata, FSIZE_XATTR_PREFIX, data_from_uint64(0));
+	if (ret) {
+		dict_unref(xdata);
+		goto error;
+	}
+	STACK_WIND(frame,
+		   crypt_readdirp_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp,
+		   fd,
+		   size,
+		   offset,
+		   xdata);
+	dict_unref(xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(readdirp, frame, -1, ret, NULL, NULL);
+	return 0;
+}
+
+static int32_t crypt_access(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc,
+			    int32_t mask, dict_t *xdata)
+{
+	gf_log(this->name, GF_LOG_WARNING,
+	       "NFS mounts of encrypted volumes are unsupported");
+	STACK_UNWIND_STRICT(access, frame, -1, EPERM, NULL);
+	return 0;
+}
+
+int32_t master_set_block_size (xlator_t *this, crypt_private_t *priv,
+			       dict_t *options)
+{
+	uint64_t block_size = 0;
+	struct master_cipher_info *master = get_master_cinfo(priv);
+
+	if (options != NULL)
+		GF_OPTION_RECONF("block-size", block_size, options,
+				 size, error);
+	else
+		GF_OPTION_INIT("block-size", block_size, size, error);
+
+	switch (block_size) {
+	case 512:
+		master->m_block_bits = 9;
+		break;
+	case 1024:
+		master->m_block_bits = 10;
+		break;
+	case 2048:
+		master->m_block_bits = 11;
+		break;
+	case 4096:
+		master->m_block_bits = 12;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_ERROR,
+		       "FATAL: unsupported block size %llu",
+		       (unsigned long long)block_size);
+		goto error;
+	}
+	return 0;
+ error:
+	return -1;
+}
+
+int32_t master_set_alg(xlator_t *this, crypt_private_t *priv)
+{
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	master->m_alg = AES_CIPHER_ALG;
+	return 0;
+}
+
+int32_t master_set_mode(xlator_t *this, crypt_private_t *priv)
+{
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	master->m_mode = XTS_CIPHER_MODE;
+	return 0;
+}
+
+/*
+ * set key size in bits to the master info
+ * Pre-conditions: cipher mode in the master info is uptodate.
+ */
+static int master_set_data_key_size (xlator_t *this, crypt_private_t *priv,
+				     dict_t *options)
+{
+	int32_t ret;
+	uint64_t key_size = 0;
+	struct master_cipher_info *master = get_master_cinfo(priv);
+
+	if (options != NULL)
+		GF_OPTION_RECONF("data-key-size", key_size, options,
+				 size, error);
+	else
+		GF_OPTION_INIT("data-key-size", key_size, size, error);
+
+	ret = data_cipher_algs[master->m_alg][master->m_mode].check_key(key_size);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "FATAL: wrong bin key size %llu for alg %d mode %d",
+		       (unsigned long long)key_size,
+		       (int)master->m_alg,
+		       (int)master->m_mode);
+		goto error;
+	}
+	master->m_dkey_size = key_size;
+	return 0;
+ error:
+	return -1;
+}
+
+static int is_hex(char *s) {
+	return  ('0' <= *s && *s <= '9') || ('a' <= *s && *s <= 'f');
+}
+
+static int parse_hex_buf(xlator_t *this, char *src, unsigned char *dst,
+			 int hex_size)
+{
+	int i;
+	int hex_byte = 0;
+
+	for (i = 0; i < (hex_size / 2); i++) {
+		if (!is_hex(src + i*2) || !is_hex(src + i*2 + 1)) {
+			gf_log("crypt", GF_LOG_ERROR,
+			       "FATAL: not hex symbol in key");
+			return -1;
+		}
+		if (sscanf(src + i*2, "%2x", &hex_byte) != 1) {
+			gf_log("crypt", GF_LOG_ERROR,
+			       "FATAL: can not parse hex key");
+			return -1;
+		}
+		dst[i] = hex_byte & 0xff;
+	}
+	return 0;
+}
+
+/*
+ * Parse options;
+ * install master volume key
+ */
+int32_t master_set_master_vol_key(xlator_t *this, crypt_private_t *priv)
+{
+	int32_t ret;
+	FILE *file = NULL;
+
+	int32_t key_size;
+	char *opt_key_file_pathname = NULL;
+
+	unsigned char bin_buf[MASTER_VOL_KEY_SIZE];
+	char hex_buf[2 * MASTER_VOL_KEY_SIZE];
+
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	/*
+	 * extract master key passed via option
+	 */
+	GF_OPTION_INIT("master-key", opt_key_file_pathname, path, bad_key);
+
+	if (!opt_key_file_pathname) {
+		gf_log(this->name, GF_LOG_ERROR, "FATAL: missing master key");
+		return -1;
+	}
+	gf_log(this->name, GF_LOG_DEBUG, "handling file key %s",
+	       opt_key_file_pathname);
+
+	file = fopen(opt_key_file_pathname, "r");
+	if (file == NULL) {
+		gf_log(this->name, GF_LOG_ERROR,
+		       "FATAL: can not open file with master key");
+		return -1;
+	}
+	/*
+	 * extract hex key
+	 */
+	key_size = fread(hex_buf, 1, sizeof(hex_buf), file);
+	if (key_size < sizeof(hex_buf)) {
+		gf_log(this->name, GF_LOG_ERROR,
+		       "FATAL: master key is too short");
+		goto bad_key;
+	}
+	ret = parse_hex_buf(this, hex_buf, bin_buf, key_size);
+	if (ret)
+		goto bad_key;
+	memcpy(master->m_key, bin_buf, MASTER_VOL_KEY_SIZE);
+	memset(hex_buf, 0, sizeof(hex_buf));
+	fclose(file);
+
+ 	memset(bin_buf, 0, sizeof(bin_buf));
+	return 0;
+ bad_key:
+	gf_log(this->name, GF_LOG_ERROR, "FATAL: bad master key");
+	if (file)
+		fclose(file);
+	memset(bin_buf, 0, sizeof(bin_buf));
+	return -1;
+}
+
+/*
+ * Derive volume key for object-id authentication
+ */
+int32_t master_set_nmtd_vol_key(xlator_t *this, crypt_private_t *priv)
+{
+	return get_nmtd_vol_key(get_master_cinfo(priv));
+}
+
+int32_t crypt_init_xlator(xlator_t *this)
+{
+	int32_t ret;
+	crypt_private_t *priv = this->private; 
+
+	ret = master_set_alg(this, priv);
+	if (ret)
+		return ret;
+	ret = master_set_mode(this, priv);
+	if (ret)
+		return ret;
+	ret = master_set_block_size(this, priv, NULL);
+	if (ret)
+		return ret;
+	ret = master_set_data_key_size(this, priv, NULL);
+	if (ret)
+		return ret;
+	ret = master_set_master_vol_key(this, priv);
+	if (ret)
+		return ret;
+	return master_set_nmtd_vol_key(this, priv);
+}
+
+static int32_t crypt_alloc_private(xlator_t *this)
+{
+	this->private = GF_CALLOC(1, sizeof(crypt_private_t), gf_crypt_mt_priv);
+	if (!this->private) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "Can not allocate memory for private data");
+		return ENOMEM;
+	}
+	return 0;
+}
+
+static void crypt_free_private(xlator_t *this)
+{
+	crypt_private_t *priv = this->private;
+	if (priv) {
+		memset(priv, 0, sizeof(*priv));
+		GF_FREE(priv);
+	}
+}
+
+int32_t reconfigure (xlator_t *this, dict_t *options)
+{
+	int32_t ret = -1;
+	crypt_private_t *priv = NULL;
+
+	GF_VALIDATE_OR_GOTO ("crypt", this, error);
+	GF_VALIDATE_OR_GOTO (this->name, this->private, error);
+	GF_VALIDATE_OR_GOTO (this->name, options, error);
+
+	priv = this->private;
+
+	ret = master_set_block_size(this, priv, options);
+	if (ret) {
+		gf_log("this->name", GF_LOG_ERROR,
+		       "Failed to reconfure block size");
+		goto error;
+	}
+	ret = master_set_data_key_size(this, priv, options);
+	if (ret) {
+		gf_log("this->name", GF_LOG_ERROR,
+		       "Failed to reconfure data key size");
+		goto error;
+	}
+	return 0;
+ error:
+	return ret;
+}
+
+int32_t init(xlator_t *this)
+{
+	int32_t ret;
+
+	if (!this->children || this->children->next) {
+		gf_log ("crypt", GF_LOG_ERROR,
+			"FATAL: crypt should have exactly one child");
+		return EINVAL;
+	}
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+	ret = crypt_alloc_private(this);
+	if (ret)
+		return ret;
+	ret = crypt_init_xlator(this);
+	if (ret)
+		goto error;
+	this->local_pool = mem_pool_new(crypt_local_t, 64);
+        if (!this->local_pool) {
+		gf_log(this->name, GF_LOG_ERROR,
+		       "failed to create local_t's memory pool");
+		ret = ENOMEM;
+                goto error;
+        }
+	gf_log ("crypt", GF_LOG_INFO, "crypt xlator loaded");
+	return 0;
+ error:
+	crypt_free_private(this);
+	return ret;
+}
+
+void fini (xlator_t *this)
+{
+	crypt_free_private(this);
+}
+
+struct xlator_fops fops = {
+	.readv        = crypt_readv,
+	.writev       = crypt_writev,
+	.truncate     = crypt_truncate,
+	.ftruncate    = crypt_ftruncate,
+	.setxattr     = crypt_setxattr,
+	.fsetxattr    = crypt_fsetxattr,
+	.link         = crypt_link,
+	.unlink       = crypt_unlink,
+	.rename       = crypt_rename,
+	.open         = crypt_open,
+	.create       = crypt_create,
+	.stat         = crypt_stat,
+	.fstat        = crypt_fstat,
+	.lookup       = crypt_lookup,
+	.readdirp     = crypt_readdirp,
+	.access       = crypt_access   
+};
+
+struct xlator_cbks cbks = {
+	.forget       = crypt_forget
+};
+
+struct volume_options options[] = {
+	{ .key = {"master-key"},
+	  .type = GF_OPTION_TYPE_PATH,
+	  .description = "Pathname of regular file which contains master volume key"
+	},
+	{ .key = {"data-key-size"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .description = "Data key size (bits)",
+	  .min = 256,
+	  .max = 512,
+	  .default_value = "256",
+	},
+	{ .key = {"block-size"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .description = "Atom size (bits)",
+	  .min = 512,
+	  .max = 4096,
+	  .default_value = "4096"
+	},
+	{ .key  = {NULL} },
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt.h b/xlators/encryption/crypt/src/crypt.h
new file mode 100644
index 000000000..01a8542ab
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt.h
@@ -0,0 +1,899 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CRYPT_H__
+#define __CRYPT_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include <openssl/aes.h>
+#include <openssl/evp.h>
+#include <openssl/sha.h>
+#include <openssl/hmac.h>
+#include <openssl/cmac.h>
+#include <openssl/modes.h>
+#include "crypt-mem-types.h"
+
+#define CRYPT_XLATOR_ID  (0)
+
+#define MAX_IOVEC_BITS (3)
+#define MAX_IOVEC (1 << MAX_IOVEC_BITS)
+#define KEY_FACTOR_BITS  (6)
+
+#define DEBUG_CRYPT (0)
+#define TRIVIAL_TFM (0)
+
+#define CRYPT_MIN_BLOCK_BITS (9)
+#define CRYPT_MAX_BLOCK_BITS (12)
+
+#define MASTER_VOL_KEY_SIZE (32)
+#define NMTD_VOL_KEY_SIZE (16)
+
+struct crypt_key {
+	uint32_t len;
+	const char *label;	
+};
+
+/*
+ * Add new key types to the end of this
+ * enumeration but before LAST_KEY_TYPE
+ */
+typedef enum {
+	MASTER_VOL_KEY,
+	NMTD_VOL_KEY,
+	NMTD_LINK_KEY,
+	EMTD_FILE_KEY,
+	DATA_FILE_KEY_256,
+	DATA_FILE_KEY_512,
+	LAST_KEY_TYPE
+}crypt_key_type;
+
+struct kderive_context {
+	const unsigned char *pkey;/* parent key */
+	uint32_t pkey_len;        /* parent key size, bits */
+	uint32_t ckey_len;        /* child key size, bits */
+	unsigned char *fid;       /* fixed input data, NIST 800-108, 5.1 */
+	uint32_t fid_len;         /* fid len, bytes */
+	unsigned char *out;       /* contains child keying material */
+	uint32_t out_len;         /* out len, bytes */
+};
+
+typedef enum {
+	DATA_ATOM,
+	HOLE_ATOM,
+	LAST_DATA_TYPE
+}atom_data_type;
+
+typedef enum {
+	HEAD_ATOM,
+	TAIL_ATOM,
+	FULL_ATOM,
+	LAST_LOCALITY_TYPE
+}atom_locality_type;
+
+typedef enum {
+	MTD_CREATE,
+	MTD_APPEND,
+	MTD_OVERWRITE,
+	MTD_CUT,
+	MTD_LAST_OP
+} mtd_op_t;
+
+struct xts128_context {
+	void      *key1, *key2;
+	block128_f block1,block2;
+};
+
+struct object_cipher_info {
+	cipher_alg_t  o_alg;
+        cipher_mode_t o_mode;
+	uint32_t      o_block_bits;
+        uint32_t      o_dkey_size; /* raw data key size in bits */
+	union {
+		struct {
+			unsigned char ivec[16];
+			AES_KEY dkey[2];
+			AES_KEY tkey;    /* key used for tweaking */
+			XTS128_CONTEXT xts;
+		} aes_xts;
+	} u;
+};
+
+struct master_cipher_info {
+	/*
+	 * attributes inherited by newly created regular files
+	 */
+	cipher_alg_t  m_alg;
+        cipher_mode_t m_mode;
+	uint32_t      m_block_bits;
+        uint32_t      m_dkey_size; /* raw key size in bits */
+	/*
+	 * master key
+	 */
+	unsigned char m_key[MASTER_VOL_KEY_SIZE];
+	/* 
+	 * volume key for oid authentication
+	 */
+	unsigned char m_nmtd_key[NMTD_VOL_KEY_SIZE];
+};
+
+/*
+* This info is not changed during file's life
+ */
+struct crypt_inode_info {
+#if DEBUG_CRYPT
+	loc_t *loc; /* pathname that the file has been
+		       opened, or created with */
+#endif
+	uint16_t nr_minor;
+	uuid_t oid;
+	struct object_cipher_info cinfo;
+};
+
+/*
+ * this should locate in secure memory
+ */
+typedef struct {
+	struct master_cipher_info master;
+} crypt_private_t;
+
+static inline struct master_cipher_info *get_master_cinfo(crypt_private_t *priv)
+{
+	return &priv->master;
+}
+
+static inline struct object_cipher_info *get_object_cinfo(struct crypt_inode_info
+							 *info)
+{
+	return &info->cinfo;
+}
+
+/*
+ * this describes layouts and properties
+ * of atoms in an aligned vector
+ */
+struct avec_config {
+	uint32_t atom_size;
+	atom_data_type type;
+	size_t orig_size;
+	off_t orig_offset;
+	size_t expanded_size;
+	off_t aligned_offset;
+
+	uint32_t off_in_head;
+	uint32_t off_in_tail;
+	uint32_t gap_in_tail;
+	uint32_t nr_full_blocks;
+
+	struct iovec *avec; /* aligned vector */
+	uint32_t acount; /* number of avec components. The same
+			  * as number of occupied logical blocks */
+	char **pool;
+	uint32_t blocks_in_pool;
+	uint32_t cursor; /* makes sense only for ordered writes,
+			  * so there is no races on this counter.
+			  *
+			  * Cursor is per-config object, we don't
+			  * reset cursor for atoms of different
+			  * localities (head, tail, full)
+			  */
+};
+
+
+typedef struct {
+	glusterfs_fop_t fop; /* code of FOP this local info built for */
+	fd_t *fd;
+	inode_t *inode;
+	loc_t *loc;
+	int32_t mac_idx;
+	loc_t *newloc;
+	int32_t flags;
+	int32_t wbflags;
+	struct crypt_inode_info *info;
+	struct iobref *iobref;
+	struct iobref *iobref_data;
+	off_t offset;
+
+	uint64_t old_file_size; /* per FOP, retrieved under lock held */
+	uint64_t cur_file_size; /* per iteration, before issuing IOs */
+	uint64_t new_file_size; /* per iteration, after issuing IOs */
+
+	uint64_t io_offset; /* offset of IOs issued per iteration */
+	uint64_t io_offset_nopad; /* offset of user's data in the atom */
+	uint32_t io_size; /* size of IOs issued per iteration */
+	uint32_t io_size_nopad; /* size of user's data in the IOs */
+	uint32_t eof_padding_size; /* size od EOF padding in the IOs */
+
+	gf_lock_t call_lock; /* protect nr_calls from many cbks */
+	int32_t nr_calls;
+
+	atom_data_type active_setup; /* which setup (hole or date)
+					is currently active */
+	/* data setup */
+	struct avec_config data_conf;
+
+	/* hole setup */
+	int hole_conv_in_proggress;
+	gf_lock_t hole_lock; /* protect hole config from many cbks */
+	int hole_handled;
+	struct avec_config hole_conf;
+	struct iatt buf;
+	struct iatt prebuf;
+	struct iatt postbuf;
+	struct iatt *prenewparent;
+	struct iatt *postnewparent;
+	int32_t op_ret;
+	int32_t op_errno;
+	int32_t rw_count; /* total read or written */
+	gf_lock_t rw_count_lock; /* protect the counter above */
+	unsigned char *format; /* for create, update format string */
+	uint32_t format_size;
+	uint32_t msgflags; /* messages for crypt_open() */
+	dict_t *xdata;
+	dict_t *xattr;
+	struct iovec vec; /* contains last file's atom for
+			     read-prune-write sequence */
+	gf_boolean_t custom_mtd;
+	/*
+	 * the next 3 fields are used by readdir and friends
+	 */
+	gf_dirent_t *de; /* directory entry */
+	char *de_path; /* pathname of directory entry */
+	uint32_t de_prefix_len; /* lenght of the parent's pathname */
+	gf_dirent_t *entries;
+
+	uint32_t update_disk_file_size:1;
+} crypt_local_t;
+
+/* This represents a (read)modify-write atom */
+struct rmw_atom {
+	atom_locality_type locality;
+	/*
+	 * read-modify-write sequence of the atom
+	 */
+	int32_t (*rmw)(call_frame_t *frame,
+		       void *cookie,
+		       xlator_t *this,
+		       int32_t op_ret,
+		       int32_t op_errno,
+		       struct iovec *vec,
+		       int32_t count,
+		       struct iatt *stbuf,
+		       struct iobref *iobref,
+		       dict_t *xdata);
+	/*
+	 * offset of the logical block in a file
+	 */
+	loff_t (*offset_at)(call_frame_t *frame,
+			    struct object_cipher_info *object);
+	/*
+	 * IO offset in an atom
+	 */
+	uint32_t (*offset_in)(call_frame_t *frame,
+			      struct object_cipher_info *object);
+	/*
+	 * number of bytes of plain text of this atom that user
+	 * wants to read/write.
+	 * It can be smaller than atom_size in the case of head
+	 * or tail atoms.
+	 */
+	uint32_t (*io_size_nopad)(call_frame_t *frame,
+				  struct object_cipher_info *object);
+	/*
+	 * which iovec represents the atom
+	 */
+	struct iovec *(*get_iovec)(call_frame_t *frame, uint32_t count);
+	/*
+	 * how many bytes of partial block should be uptodated by
+	 * reading from disk.
+	 * This is used to perform a read component of RMW (read-modify-write).
+	 */
+	uint32_t (*count_to_uptodate)(call_frame_t *frame, struct object_cipher_info *object);
+	struct avec_config *(*get_config)(call_frame_t *frame);
+};
+
+struct data_cipher_alg {
+	gf_boolean_t atomic; /* true means that algorithm requires
+				to pad data before cipher transform */
+	gf_boolean_t should_pad; /* true means that algorithm requires
+				    to pad the end of file with extra-data */
+	uint32_t blkbits; /* blksize = 1 << blkbits */
+	/*
+	 * any preliminary sanity checks goes here
+	 */
+	int32_t (*init)(void);
+	/*
+	 * set alg-mode specific inode info
+	 */
+	int32_t (*set_private)(struct crypt_inode_info *info,
+			       struct master_cipher_info *master);
+	/*
+	 * check alg-mode specific data key
+	 */
+	int32_t (*check_key)(uint32_t key_size);
+	void (*set_iv)(off_t offset, struct object_cipher_info *object);
+	int32_t (*encrypt)(const unsigned char *from, unsigned char *to,
+			   size_t length, off_t offset, const int enc,
+			   struct object_cipher_info *object);
+};
+
+/*
+ * version-dependent metadata loader
+ */
+struct crypt_mtd_loader {
+	/*
+	 * return core format size
+	 */
+	size_t (*format_size)(mtd_op_t op, size_t old_size);
+	/*
+	 * pack version-specific metadata of an object
+	 * at ->create()
+	 */
+	int32_t (*create_format)(unsigned char *wire,
+				 loc_t *loc,
+				 struct crypt_inode_info *info,
+				 struct master_cipher_info *master);
+	/*
+	 * extract version-specific metadata of an object
+	 * at ->open() time
+	 */
+	int32_t (*open_format)(unsigned char *wire,
+			       int32_t len,
+			       loc_t *loc,
+			       struct crypt_inode_info *info,
+			       struct master_cipher_info *master,
+			       crypt_local_t *local,
+			       gf_boolean_t load_info);
+	int32_t (*update_format)(unsigned char *new,
+				 unsigned char *old,
+				 size_t old_len,
+				 int32_t mac_idx,
+				 mtd_op_t op,
+				 loc_t *loc,
+				 struct crypt_inode_info *info,
+				 struct master_cipher_info *master,
+				 crypt_local_t *local);
+};
+
+typedef int32_t (*end_writeback_handler_t)(call_frame_t *frame,
+					   void *cookie,
+					   xlator_t *this,
+					   int32_t op_ret,
+					   int32_t op_errno,
+					   struct iatt *prebuf,
+					   struct iatt *postbuf,
+					   dict_t *xdata);
+typedef void (*linkop_wind_handler_t)(call_frame_t *frame, xlator_t *this);
+typedef void (*linkop_unwind_handler_t)(call_frame_t *frame);
+
+
+/* Declarations */
+
+/* keys.c */
+extern struct crypt_key crypt_keys[LAST_KEY_TYPE];
+int32_t get_nmtd_vol_key(struct master_cipher_info *master);
+int32_t get_nmtd_link_key(loc_t *loc,
+			  struct master_cipher_info *master,
+			  unsigned char *result);
+int32_t get_emtd_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  unsigned char *result);
+int32_t get_data_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  uint32_t keysize,
+			  unsigned char *key);
+/* data.c */
+extern struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE];
+void encrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off);
+void decrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off);
+int32_t align_iov_by_atoms(xlator_t *this,
+			   crypt_local_t *local,
+			   struct object_cipher_info *object,
+			   struct iovec *vec /* input vector */,
+			   int32_t count /* number of vec components */,
+			   struct iovec *avec /* aligned vector */,
+			   char **blocks /* pool of blocks */,
+			   uint32_t *blocks_allocated,
+			   struct avec_config *conf);
+int32_t set_config_avec_data(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     struct iovec *vec,
+			     int32_t vec_count);
+int32_t set_config_avec_hole(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     glusterfs_fop_t fop);
+void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
+		    struct avec_config *conf, atom_data_type dtype);
+void set_config_offsets(call_frame_t *frame,
+			xlator_t *this,
+			uint64_t offset,
+			uint64_t count,
+			atom_data_type dtype,
+			int32_t setup_gap_in_tail);
+
+/* metadata.c */
+extern struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER];
+
+int32_t alloc_format(crypt_local_t *local, size_t size);
+int32_t alloc_format_create(crypt_local_t *local);
+void free_format(crypt_local_t *local);
+size_t format_size(mtd_op_t op, size_t old_size);
+size_t new_format_size(void);
+int32_t open_format(unsigned char *str, int32_t len, loc_t *loc,
+		    struct crypt_inode_info *info,
+		    struct master_cipher_info *master, crypt_local_t *local,
+		    gf_boolean_t load_info);
+int32_t update_format(unsigned char *new, unsigned char *old,
+		      size_t old_len, int32_t mac_idx, mtd_op_t op, loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master,
+		      crypt_local_t *local);
+int32_t create_format(unsigned char *wire,
+		      loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master);
+
+/* atom.c */
+struct rmw_atom *atom_by_types(atom_data_type data,
+			       atom_locality_type locality);
+void submit_partial(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    atom_locality_type ltype);
+void submit_full(call_frame_t *frame, xlator_t *this);
+
+/* crypt.c */
+
+end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop);
+static size_t iovec_get_size(struct iovec *vec, uint32_t count);
+void set_local_io_params_writev(call_frame_t *frame,
+				struct object_cipher_info *object,
+				struct rmw_atom *atom, off_t io_offset,
+				uint32_t io_size);
+void link_wind(call_frame_t *frame, xlator_t *this);
+void unlink_wind(call_frame_t *frame, xlator_t *this);
+void link_unwind(call_frame_t *frame);
+void unlink_unwind(call_frame_t *frame);
+void rename_wind(call_frame_t *frame, xlator_t *this);
+void rename_unwind(call_frame_t *frame);
+
+/* Inline functions */
+
+static inline size_t iovec_get_size(struct iovec *vec, uint32_t count)
+{
+	int i;
+	size_t size = 0;
+	for (i = 0; i < count; i++)
+		size += vec[i].iov_len;
+	return size;
+}
+
+static inline int32_t crypt_xlator_id(void)
+{
+	return CRYPT_XLATOR_ID;
+}
+
+static inline mtd_loader_id current_mtd_loader(void)
+{
+	return MTD_LOADER_V1;
+}
+
+static inline uint32_t master_key_size (void)
+{
+	return crypt_keys[MASTER_VOL_KEY].len >> 3;
+}
+
+static inline uint32_t nmtd_vol_key_size (void)
+{
+	return crypt_keys[NMTD_VOL_KEY].len >> 3;
+}
+
+static inline uint32_t alg_mode_blkbits(cipher_alg_t alg,
+					cipher_mode_t mode)
+{
+	return data_cipher_algs[alg][mode].blkbits;
+}
+
+static inline uint32_t alg_mode_blksize(cipher_alg_t alg,
+					cipher_mode_t mode)
+{
+	return 1 << alg_mode_blkbits(alg, mode);
+}
+
+static inline gf_boolean_t alg_mode_atomic(cipher_alg_t alg,
+					   cipher_mode_t mode)
+{
+	return data_cipher_algs[alg][mode].atomic;
+}
+
+static inline gf_boolean_t alg_mode_should_pad(cipher_alg_t alg,
+					       cipher_mode_t mode)
+{
+	return data_cipher_algs[alg][mode].should_pad;
+}
+
+static inline uint32_t master_alg_blksize(struct master_cipher_info *mr)
+{
+	return alg_mode_blksize(mr->m_alg, mr->m_mode);
+}
+
+static inline uint32_t master_alg_blkbits(struct master_cipher_info *mr)
+{
+	return alg_mode_blkbits(mr->m_alg, mr->m_mode);
+}
+
+static inline gf_boolean_t master_alg_atomic(struct master_cipher_info *mr)
+{
+	return alg_mode_atomic(mr->m_alg, mr->m_mode);
+}
+
+static inline gf_boolean_t master_alg_should_pad(struct master_cipher_info *mr)
+{
+	return alg_mode_should_pad(mr->m_alg, mr->m_mode);
+}
+
+static inline uint32_t object_alg_blksize(struct object_cipher_info *ob)
+{
+	return alg_mode_blksize(ob->o_alg, ob->o_mode);
+}
+
+static inline uint32_t object_alg_blkbits(struct object_cipher_info *ob)
+{
+	return alg_mode_blkbits(ob->o_alg, ob->o_mode);
+}
+
+static inline gf_boolean_t object_alg_atomic(struct object_cipher_info *ob)
+{
+	return alg_mode_atomic(ob->o_alg, ob->o_mode);
+}
+
+static inline gf_boolean_t object_alg_should_pad(struct object_cipher_info *ob)
+{
+	return alg_mode_should_pad(ob->o_alg, ob->o_mode);
+}
+
+static inline uint32_t aes_raw_key_size(struct master_cipher_info *master)
+{
+	return master->m_dkey_size >> 3;
+}
+
+static inline struct avec_config *get_hole_conf(call_frame_t *frame)
+{
+	return &(((crypt_local_t *)frame->local)->hole_conf);
+}
+
+static inline struct avec_config *get_data_conf(call_frame_t *frame)
+{
+	return &(((crypt_local_t *)frame->local)->data_conf);
+}
+
+static inline int32_t get_atom_bits (struct object_cipher_info *object)
+{
+	return object->o_block_bits;
+}
+
+static inline int32_t get_atom_size (struct object_cipher_info *object)
+{
+	return 1 << get_atom_bits(object);
+}
+
+static inline int32_t has_head_block(struct avec_config *conf)
+{
+	return conf->off_in_head ||
+		(conf->acount == 1 && conf->off_in_tail);
+}
+
+static inline int32_t has_tail_block(struct avec_config *conf)
+{
+	return conf->off_in_tail && conf->acount > 1;
+}
+
+static inline int32_t has_full_blocks(struct avec_config *conf)
+{
+	return conf->nr_full_blocks;
+}
+
+static inline int32_t should_submit_head_block(struct avec_config *conf)
+{
+	return has_head_block(conf) && (conf->cursor == 0);
+}
+
+static inline int32_t should_submit_tail_block(struct avec_config *conf)
+{
+	return has_tail_block(conf) && (conf->cursor == conf->acount - 1);
+}
+
+static inline int32_t should_submit_full_block(struct avec_config *conf)
+{
+	uint32_t start = has_head_block(conf) ? 1 : 0;
+
+	return has_full_blocks(conf) &&
+		conf->cursor >= start &&
+		conf->cursor < start + conf->nr_full_blocks;
+}
+
+#if DEBUG_CRYPT
+static inline void crypt_check_input_len(size_t len,
+					 struct object_cipher_info *object)
+{
+	if (object_alg_should_pad(object) && (len & (object_alg_blksize(object) - 1)))
+		gf_log ("crypt", GF_LOG_DEBUG, "bad input len: %d", (int)len);
+}
+
+static inline void check_head_block(struct avec_config *conf)
+{
+	if (!has_head_block(conf))
+		gf_log("crypt", GF_LOG_DEBUG, "not a head atom");
+}
+
+static inline void check_tail_block(struct avec_config *conf)
+{
+	if (!has_tail_block(conf))
+		gf_log("crypt", GF_LOG_DEBUG, "not a tail atom");
+}
+
+static inline void check_full_block(struct avec_config *conf)
+{
+	if (!has_full_blocks(conf))
+		gf_log("crypt", GF_LOG_DEBUG, "not a full atom");
+}
+
+static inline void check_cursor_head(struct avec_config *conf)
+{
+	if (!has_head_block(conf))
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Illegal call of head atom method");
+	else if (conf->cursor != 0)
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Cursor (%d) is not at head atom",
+		       conf->cursor);
+}
+
+static inline void check_cursor_full(struct avec_config *conf)
+{
+	if (!has_full_blocks(conf))
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Illegal call of full atom method");
+	if (has_head_block(conf) && (conf->cursor == 0))
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Cursor is not at full atom");
+}
+
+/*
+ * FIXME: use avec->iov_len to check setup
+ */
+static inline int data_local_invariant(crypt_local_t *local)
+{
+	return 0;
+}
+
+#else
+#define crypt_check_input_len(len, object) noop
+#define check_head_block(conf) noop
+#define check_tail_block(conf) noop
+#define check_full_block(conf) noop
+#define check_cursor_head(conf) noop
+#define check_cursor_full(conf) noop
+
+#endif /* DEBUG_CRYPT */
+
+static inline struct avec_config *conf_by_type(call_frame_t *frame,
+					       atom_data_type dtype)
+{
+	struct avec_config *conf = NULL;
+
+	switch (dtype) {
+	case HOLE_ATOM:
+		conf = get_hole_conf(frame);
+		break;
+	case DATA_ATOM:
+		conf = get_data_conf(frame);
+		break;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad atom type");
+	}
+	return conf;
+}
+
+static inline uint32_t nr_calls_head(struct avec_config *conf)
+{
+	return has_head_block(conf) ? 1 : 0;
+}
+
+static inline uint32_t nr_calls_tail(struct avec_config *conf)
+{
+	return has_tail_block(conf) ? 1 : 0;
+}
+
+static inline uint32_t nr_calls_full(struct avec_config *conf)
+{
+	switch(conf->type) {
+	case HOLE_ATOM:
+		return has_full_blocks(conf);
+	case DATA_ATOM:
+		return has_full_blocks(conf) ?
+			logical_blocks_occupied(0,
+						conf->nr_full_blocks,
+						MAX_IOVEC_BITS)	: 0;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad atom data type");
+		return 0;
+	}
+}
+
+static inline uint32_t nr_calls(struct avec_config *conf)
+{
+	return nr_calls_head(conf) + nr_calls_tail(conf) + nr_calls_full(conf);
+}
+
+static inline uint32_t nr_calls_data(call_frame_t *frame)
+{
+	return nr_calls(get_data_conf(frame));
+}
+
+static inline uint32_t nr_calls_hole(call_frame_t *frame)
+{
+	return nr_calls(get_hole_conf(frame));
+}
+
+static inline void get_one_call_nolock(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+
+	++local->nr_calls;
+
+	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", 1);
+}
+
+static inline void get_one_call(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+
+	LOCK(&local->call_lock);
+	get_one_call_nolock(frame);
+	UNLOCK(&local->call_lock);
+}
+
+static inline void get_nr_calls_nolock(call_frame_t *frame, int32_t nr)
+{
+	crypt_local_t *local = frame->local;
+
+	local->nr_calls += nr;
+
+	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", nr);
+}
+
+static inline void get_nr_calls(call_frame_t *frame, int32_t nr)
+{
+	crypt_local_t *local = frame->local;
+
+	LOCK(&local->call_lock);
+	get_nr_calls_nolock(frame, nr);
+	UNLOCK(&local->call_lock);
+}
+
+static inline int put_one_call(crypt_local_t *local)
+{
+	uint32_t last = 0;
+
+	LOCK(&local->call_lock);
+	if (--local->nr_calls == 0)
+		last = 1;
+
+	//gf_log("crypt", GF_LOG_DEBUG, "put %d calls", 1);
+
+	UNLOCK(&local->call_lock);
+	return last;
+}
+
+static inline int is_appended_write(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->orig_offset + conf->orig_size > local->old_file_size;
+}
+
+static inline int is_ordered_mode(call_frame_t *frame)
+{
+#if 0
+	crypt_local_t *local = frame->local;
+	return local->fop == GF_FOP_FTRUNCATE ||
+		(local->fop == GF_FOP_WRITE && is_appended_write(frame));
+#endif
+	return 1;
+}
+
+static inline int32_t hole_conv_completed(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+	return conf->cursor == conf->acount;
+}
+
+static inline int32_t data_write_in_progress(crypt_local_t *local)
+{
+	return local->active_setup == DATA_ATOM;
+}
+
+static inline int32_t parent_is_crypt_xlator(call_frame_t *frame,
+					     xlator_t *this)
+{
+	return frame->parent->this == this;
+}
+
+static inline linkop_wind_handler_t linkop_wind_dispatch(glusterfs_fop_t fop)
+{
+	switch(fop){
+	case GF_FOP_LINK:
+		return link_wind;
+	case GF_FOP_UNLINK:
+		return unlink_wind;
+	case GF_FOP_RENAME:
+		return rename_wind;
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
+		return NULL;
+	}
+}
+
+static inline linkop_unwind_handler_t linkop_unwind_dispatch(glusterfs_fop_t fop)
+{
+	switch(fop){
+	case GF_FOP_LINK:
+		return link_unwind;
+	case GF_FOP_UNLINK:
+		return unlink_unwind;
+	case GF_FOP_RENAME:
+		return rename_unwind;
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
+		return NULL;		
+	}	
+}
+
+static inline mtd_op_t linkop_mtdop_dispatch(glusterfs_fop_t fop)
+{
+	switch (fop) {
+	case GF_FOP_LINK:
+		return MTD_APPEND;
+	case GF_FOP_UNLINK:
+		return MTD_CUT;
+	case GF_FOP_RENAME:
+		return MTD_OVERWRITE;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad link operation %d", fop);
+		return MTD_LAST_OP;
+	}
+}
+
+#endif /* __CRYPT_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/data.c b/xlators/encryption/crypt/src/data.c
new file mode 100644
index 000000000..762fa554a
--- /dev/null
+++ b/xlators/encryption/crypt/src/data.c
@@ -0,0 +1,769 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+static void set_iv_aes_xts(off_t offset, struct object_cipher_info *object)
+{
+	unsigned char *ivec;
+
+	ivec = object->u.aes_xts.ivec;
+
+	/* convert the tweak into a little-endian byte
+	 * array (IEEE P1619/D16, May 2007, section 5.1)
+	 */
+
+	*((uint64_t *)ivec) = htole64(offset);
+
+	/* ivec is padded with zeroes */
+}
+
+static int32_t aes_set_keys_common(unsigned char *raw_key, uint32_t key_size,
+				   AES_KEY *keys)
+{
+	int32_t ret;
+
+	ret = AES_set_encrypt_key(raw_key,
+				  key_size,
+				  &keys[AES_ENCRYPT]);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set encrypt key failed");
+		return ret;
+	}
+	ret = AES_set_decrypt_key(raw_key,
+				  key_size,
+				  &keys[AES_DECRYPT]);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set decrypt key failed");
+		return ret;
+	}
+	return 0;
+}
+
+/*
+ * set private cipher info for xts mode
+ */
+static int32_t set_private_aes_xts(struct crypt_inode_info *info,
+				   struct master_cipher_info *master)
+{
+	int ret;
+	struct object_cipher_info *object = get_object_cinfo(info);
+	unsigned char *data_key;
+	uint32_t subkey_size;
+
+	/* init tweak value */
+	memset(object->u.aes_xts.ivec, 0, 16);
+
+	data_key = GF_CALLOC(1, object->o_dkey_size, gf_crypt_mt_key);
+	if (!data_key)
+		return ENOMEM;
+
+	/*
+	 * retrieve data keying meterial
+	 */
+	ret = get_data_file_key(info, master, object->o_dkey_size, data_key);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Failed to retrieve data key");
+		GF_FREE(data_key);
+		return ret;
+	}
+	/*
+	 * parse compound xts key
+	 */
+	subkey_size = object->o_dkey_size >> 4; /* (xts-key-size-in-bytes / 2) */
+	/*
+	 * install key for data encryption
+	 */
+	ret = aes_set_keys_common(data_key,
+				  subkey_size << 3, object->u.aes_xts.dkey);
+	if (ret) {
+		GF_FREE(data_key);
+		return ret;
+	}
+	/*
+	 * set up key used to encrypt tweaks
+	 */
+	ret = AES_set_encrypt_key(data_key + subkey_size,
+				  object->o_dkey_size / 2,
+				  &object->u.aes_xts.tkey);
+	if (ret < 0)
+		gf_log("crypt", GF_LOG_ERROR, "Set tweak key failed");
+
+	GF_FREE(data_key);
+	return ret;
+}
+
+static int32_t aes_xts_init(void)
+{
+	cassert(AES_BLOCK_SIZE == (1 << AES_BLOCK_BITS));
+	return 0;
+}
+
+static int32_t check_key_aes_xts(uint32_t keysize)
+{
+	switch(keysize) {
+	case 256:
+	case 512:
+		return 0;
+	default:
+		break;
+	}
+	return -1;
+}
+
+static int32_t encrypt_aes_xts(const unsigned char *from,
+			       unsigned char *to, size_t length,
+			       off_t offset, const int enc,
+			       struct object_cipher_info *object)
+{
+	XTS128_CONTEXT ctx;
+	if (enc) {
+		ctx.key1 = &object->u.aes_xts.dkey[AES_ENCRYPT];
+		ctx.block1 = (block128_f)AES_encrypt;
+	}
+	else {
+		ctx.key1 = &object->u.aes_xts.dkey[AES_DECRYPT];
+		ctx.block1 = (block128_f)AES_decrypt;
+	}
+	ctx.key2 = &object->u.aes_xts.tkey;
+	ctx.block2 = (block128_f)AES_encrypt;
+
+	return CRYPTO_xts128_encrypt(&ctx,
+				     object->u.aes_xts.ivec,
+				     from,
+				     to,
+				     length, enc);
+}
+
+/*
+ * Cipher input chunk @from of length @len;
+ * @to: result of cipher transform;
+ * @off: offset in a file (must be cblock-aligned);
+ */
+static void cipher_data(struct object_cipher_info *object,
+			char *from,
+			char *to,
+			off_t off,
+			size_t len,
+			const int enc)
+{
+	crypt_check_input_len(len, object);
+
+#if TRIVIAL_TFM && DEBUG_CRYPT
+	return;
+#endif
+	data_cipher_algs[object->o_alg][object->o_mode].set_iv(off, object);
+	data_cipher_algs[object->o_alg][object->o_mode].encrypt
+		((const unsigned char *)from,
+		 (unsigned char *)to,
+		 len,
+		 off,
+		 enc,
+		 object);
+}
+
+#define MAX_CIPHER_CHUNK (1 << 30)
+
+/*
+ * Do cipher (encryption/decryption) transform of a
+ * continuous region of memory.
+ *
+ * @len: a number of bytes to transform;
+ * @buf: data to transform;
+ * @off: offset in a file, should be block-aligned
+ *       for atomic cipher modes and ksize-aligned
+ *       for other modes).
+ * @dir: direction of transform (encrypt/decrypt).
+ */
+static void cipher_region(struct object_cipher_info *object,
+			  char *from,
+			  char *to,
+			  off_t off,
+			  size_t len,
+			  int dir)
+{
+	while (len > 0) {
+		size_t to_cipher;
+
+		to_cipher = len;
+		if (to_cipher > MAX_CIPHER_CHUNK)
+			to_cipher = MAX_CIPHER_CHUNK;
+
+		/* this will reset IV */
+		cipher_data(object,
+			    from,
+			    to,
+			    off,
+			    to_cipher,
+			    dir);
+		from += to_cipher;
+		to   += to_cipher;
+		off  += to_cipher;
+		len  -= to_cipher;
+	}
+}
+
+/*
+ * Do cipher transform (encryption/decryption) of
+ * plaintext/ciphertext represented by @vec.
+ *
+ * Pre-conditions: @vec represents a continuous piece
+ * of data in a file at offset @off to be ciphered
+ * (encrypted/decrypted).
+ * @count is the number of vec's components. All the
+ * components must be block-aligned, the caller is
+ * responsible for this. @dir is "direction" of
+ * transform (encrypt/decrypt).
+ */
+static void cipher_aligned_iov(struct object_cipher_info *object,
+			       struct iovec *vec,
+			       int count,
+			       off_t off,
+			       int32_t dir)
+{
+	int i;
+	int len = 0;
+
+	for (i = 0; i < count; i++) {
+		cipher_region(object,
+			      vec[i].iov_base,
+			      vec[i].iov_base,
+			      off + len,
+			      vec[i].iov_len,
+			      dir);
+		len += vec[i].iov_len;
+	}
+}
+
+void encrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off)
+{
+	cipher_aligned_iov(object, vec, count, off, 1);
+}
+
+void decrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off)
+{
+	cipher_aligned_iov(object, vec, count, off, 0);
+}
+
+#if DEBUG_CRYPT
+static void compound_stream(struct iovec *vec, int count, char *buf, off_t skip)
+{
+	int i;
+	int off = 0;
+	for (i = 0; i < count; i++) {
+		memcpy(buf + off,
+		       vec[i].iov_base + skip,
+		       vec[i].iov_len - skip);
+
+		off += (vec[i].iov_len - skip);
+		skip = 0;
+	}
+}
+
+static void check_iovecs(struct iovec *vec, int cnt,
+			 struct iovec *avec, int acnt, uint32_t off_in_head)
+{
+	char *s1, *s2;
+	uint32_t size, asize;
+
+	size = iovec_get_size(vec, cnt);
+	asize = iovec_get_size(avec, acnt) - off_in_head;
+	if (size != asize) {
+		gf_log("crypt", GF_LOG_DEBUG, "size %d is not eq asize %d",
+		       size, asize);
+		return;
+	}
+	s1 = GF_CALLOC(1, size, gf_crypt_mt_data);
+	if (!s1) {
+		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
+		return;
+	}
+	s2 = GF_CALLOC(1, asize, gf_crypt_mt_data);
+	if (!s2) {
+		GF_FREE(s1);
+		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
+		return;
+	}
+	compound_stream(vec, cnt, s1, 0);
+	compound_stream(avec, acnt, s2, off_in_head);
+	if (memcmp(s1, s2, size))
+		gf_log("crypt", GF_LOG_DEBUG, "chunks of different data");
+	GF_FREE(s1);
+	GF_FREE(s2);
+}
+
+#else
+#define check_iovecs(vec, count, avec, avecn, off) noop
+#endif /* DEBUG_CRYPT */
+
+static char *data_alloc_block(xlator_t *this, crypt_local_t *local,
+			      int32_t block_size)
+{
+	struct iobuf *iobuf = NULL;
+
+	iobuf = iobuf_get2(this->ctx->iobuf_pool, block_size);
+	if (!iobuf) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "Failed to get iobuf");
+		return NULL;
+	}
+	if (!local->iobref_data) {
+		local->iobref_data = iobref_new();
+		if (!local->iobref_data) {
+			gf_log("crypt", GF_LOG_ERROR,
+			       "Failed to get iobref");
+			iobuf_unref(iobuf);
+			return NULL;
+		}
+	}
+	iobref_add(local->iobref_data, iobuf);
+	return iobuf->ptr;
+}
+
+/*
+ * Compound @avec, which represent the same data
+ * chunk as @vec, but has aligned components of
+ * specified block size. Alloc blocks, if needed.
+ * In particular, incomplete head and tail blocks
+ * must be allocated.
+ * Put number of allocated blocks to @num_blocks.
+ *
+ * Example:
+ *
+ * input: data chunk represented by 4 components
+ * [AB],[BC],[CD],[DE];
+ * output: 5 logical blocks (0, 1, 2, 3, 4).
+ *
+ *   A     B       C     D               E
+ *   *-----*+------*-+---*----+--------+-*
+ *   |     ||      | |   |    |        | |
+ * *-+-----+*------+-*---+----*--------*-+------*
+ * 0        1        2        3        4
+ *
+ * 0 - incomplete compound (head);
+ * 1, 2 - full compound;
+ * 3 - full non-compound (the case of reuse);
+ * 4 - incomplete non-compound (tail).
+ */
+int32_t align_iov_by_atoms(xlator_t *this,
+			   crypt_local_t *local,
+			   struct object_cipher_info *object,
+			   struct iovec *vec /* input vector */,
+			   int32_t count /* number of vec components */,
+			   struct iovec *avec /* aligned vector */,
+			   char **blocks /* pool of blocks */,
+			   uint32_t *blocks_allocated,
+			   struct avec_config *conf)
+{
+	int vecn = 0;      /* number of the current component in vec */
+	int avecn = 0;     /* number of the current component in avec */
+	off_t vec_off = 0; /* offset in the current vec component,
+			    * i.e. the number of bytes have already
+			    * been copied */
+	int32_t block_size = get_atom_size(object);
+	size_t to_process; /* number of vec's bytes to copy and(or) re-use */
+	int32_t off_in_head = conf->off_in_head;
+
+	to_process = iovec_get_size(vec, count);
+
+	while (to_process > 0) {
+		if (off_in_head ||
+		    vec[vecn].iov_len - vec_off < block_size) {
+			/*
+			 * less than block_size:
+			 * the case of incomplete (head or tail),
+			 * or compound block
+			 */
+			size_t copied = 0;
+			/*
+			 * populate the pool with a new block
+			 */
+			blocks[*blocks_allocated] = data_alloc_block(this,
+								    local,
+								    block_size);
+			if (!blocks[*blocks_allocated])
+				return -ENOMEM;
+			memset(blocks[*blocks_allocated], 0, off_in_head);
+			/*
+			 * fill the block with vec components
+			 */
+			do {
+				size_t to_copy;
+
+				to_copy = vec[vecn].iov_len - vec_off;
+				if (to_copy > block_size - off_in_head)
+					to_copy = block_size - off_in_head;
+
+				memcpy(blocks[*blocks_allocated] + off_in_head + copied,
+				       vec[vecn].iov_base + vec_off,
+				       to_copy);
+
+				copied += to_copy;
+				to_process -= to_copy;
+
+				vec_off += to_copy;
+				if (vec_off == vec[vecn].iov_len) {
+					/* finished with this vecn */
+					vec_off = 0;
+					vecn++;
+				}
+			} while (copied < (block_size - off_in_head) && to_process > 0);
+			/*
+			 * update avec
+			 */
+			avec[avecn].iov_len = off_in_head + copied;
+			avec[avecn].iov_base = blocks[*blocks_allocated];
+
+			(*blocks_allocated)++;
+			off_in_head = 0;
+		} else {
+			/*
+			 * the rest of the current vec component
+			 * is not less than block_size, so reuse
+			 * the memory buffer of the component.
+			 */
+			size_t to_reuse;
+			to_reuse = (to_process > block_size ?
+				    block_size :
+				    to_process);
+			avec[avecn].iov_len = to_reuse;
+			avec[avecn].iov_base = vec[vecn].iov_base + vec_off;
+
+			vec_off += to_reuse;
+			if (vec_off == vec[vecn].iov_len) {
+				/* finished with this vecn */
+				vec_off = 0;
+				vecn++;
+			}
+			to_process -= to_reuse;
+		}
+		avecn++;
+	}
+	check_iovecs(vec, count, avec, avecn, conf->off_in_head);
+	return 0;
+}
+
+/*
+ * allocate and setup aligned vector for data submission
+ * Pre-condition: @conf is set.
+ */
+int32_t set_config_avec_data(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     struct iovec *vec,
+			     int32_t vec_count)
+{
+	int32_t ret = ENOMEM;
+	struct iovec *avec;
+	char **pool;
+	uint32_t blocks_in_pool = 0;
+
+	conf->type = DATA_ATOM;
+
+	avec = GF_CALLOC(conf->acount, sizeof(*avec), gf_crypt_mt_iovec);
+	if (!avec)
+		return ret;
+	pool = GF_CALLOC(conf->acount, sizeof(pool), gf_crypt_mt_char);
+	if (!pool) {
+		GF_FREE(avec);
+		return ret;
+	}
+	if (!vec) {
+		/*
+		 * degenerated case: no data
+		 */
+		pool[0] = data_alloc_block(this, local, get_atom_size(object));
+		if (!pool[0])
+			goto free;
+		blocks_in_pool = 1;
+		avec->iov_base = pool[0];
+		avec->iov_len = conf->off_in_tail;
+	}
+	else {
+		ret = align_iov_by_atoms(this, local, object, vec, vec_count,
+					 avec, pool, &blocks_in_pool, conf);
+		if (ret)
+			goto free;
+	}
+	conf->avec = avec;
+	conf->pool = pool;
+	conf->blocks_in_pool = blocks_in_pool;
+	return 0;
+ free:
+	GF_FREE(avec);
+	GF_FREE(pool);
+	return ret;
+}
+
+/*
+ * allocate and setup aligned vector for hole submission
+ */
+int32_t set_config_avec_hole(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     glusterfs_fop_t fop)
+{
+	uint32_t i, idx;
+	struct iovec *avec;
+	char **pool;
+	uint32_t num_blocks;
+	uint32_t blocks_in_pool = 0;
+
+	conf->type = HOLE_ATOM;
+
+	num_blocks = conf->acount -
+		(conf->nr_full_blocks ? conf->nr_full_blocks - 1 : 0);
+
+	switch (fop) {
+	case GF_FOP_WRITE:
+		/*
+		 * hole goes before data
+		 */
+		if (num_blocks == 1 && conf->off_in_tail != 0)
+		/*
+		 * we won't submit a hole which fits into
+		 * a data atom: this part of hole will be
+		 * submitted with data write
+		 */
+			return 0;
+		break;
+	case GF_FOP_FTRUNCATE:
+		/*
+		 * expanding truncate, hole goes after data,
+		 * and will be submited in any case.
+ 		 */
+		break;
+	default:
+		gf_log("crypt", GF_LOG_WARNING,
+		       "bad file operation %d", fop);
+		return 0;
+	}
+	avec = GF_CALLOC(num_blocks, sizeof(*avec), gf_crypt_mt_iovec);
+	if (!avec)
+		return ENOMEM;
+	pool = GF_CALLOC(num_blocks, sizeof(pool), gf_crypt_mt_char);
+	if (!pool) {
+		GF_FREE(avec);
+		return ENOMEM;
+	}
+	for (i = 0; i < num_blocks; i++) {
+		pool[i] = data_alloc_block(this, local, get_atom_size(object));
+		if (pool[i] == NULL)
+			goto free;
+		blocks_in_pool++;
+	}
+	if (has_head_block(conf)) {
+		/* set head block */
+		idx = 0;
+		avec[idx].iov_base = pool[idx];
+		avec[idx].iov_len = get_atom_size(object);
+		memset(avec[idx].iov_base + conf->off_in_head,
+		       0,
+		       get_atom_size(object) - conf->off_in_head);
+	}
+	if (has_tail_block(conf)) {
+		/* set tail block */
+		idx = num_blocks - 1;
+		avec[idx].iov_base = pool[idx];
+		avec[idx].iov_len = get_atom_size(object);
+		memset(avec[idx].iov_base, 0, conf->off_in_tail);
+	}
+	if (has_full_blocks(conf)) {
+		/* set full block */
+		idx = conf->off_in_head ? 1 : 0;
+		avec[idx].iov_base = pool[idx];
+		avec[idx].iov_len = get_atom_size(object);
+		/*
+		 * since we re-use the buffer,
+		 * zeroes will be set every time
+		 * before encryption, see submit_full()
+		 */
+	}
+	conf->avec = avec;
+	conf->pool = pool;
+	conf->blocks_in_pool = blocks_in_pool;
+	return 0;
+ free:
+	GF_FREE(avec);
+	GF_FREE(pool);
+	return ENOMEM;
+}
+
+/* A helper for setting up config of partial atoms (which
+ * participate in read-modify-write sequence).
+ *
+ * Calculate and setup precise amount of "extra-bytes"
+ * that should be uptodated at the end of partial (not
+ * necessarily tail!) block.
+ *
+ * Pre-condition: local->old_file_size is valid!
+ * @conf contains setup, which is enough for correct calculation
+ * of has_tail_block(), ->get_offset().
+ */
+void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
+		    struct avec_config *conf, atom_data_type dtype)
+{
+	uint32_t to_block;
+	crypt_local_t *local = frame->local;
+	uint64_t old_file_size = local->old_file_size;
+	struct rmw_atom *partial = atom_by_types(dtype,
+						 has_tail_block(conf) ?
+						 TAIL_ATOM : HEAD_ATOM);
+
+	if (old_file_size <= partial->offset_at(frame, object))
+		to_block = 0;
+	else {
+		to_block = old_file_size - partial->offset_at(frame, object);
+		if (to_block > get_atom_size(object))
+			to_block = get_atom_size(object);
+	}
+	if (to_block > conf->off_in_tail)
+		conf->gap_in_tail = to_block - conf->off_in_tail;
+	else
+		/*
+		 * nothing to uptodate
+		 */
+		conf->gap_in_tail = 0;
+}
+
+/*
+ * fill struct avec_config with offsets layouts
+ */
+void set_config_offsets(call_frame_t *frame,
+			xlator_t *this,
+			uint64_t offset,
+			uint64_t count,
+			atom_data_type dtype,
+			int32_t set_gap)
+{
+	crypt_local_t *local;
+	struct object_cipher_info *object;
+	struct avec_config *conf;
+	uint32_t resid;
+
+	uint32_t atom_size;
+	uint32_t atom_bits;
+
+	size_t orig_size;
+	off_t orig_offset;
+	size_t expanded_size;
+	off_t aligned_offset;
+
+	uint32_t off_in_head = 0;
+	uint32_t off_in_tail = 0;
+	uint32_t nr_full_blocks;
+	int32_t size_full_blocks;
+
+	uint32_t acount; /* number of alifned components to write.
+			  * The same as number of occupied logical
+			  * blocks (atoms)
+			  */
+	local = frame->local;
+	object = &local->info->cinfo;
+	conf = (dtype == DATA_ATOM ?
+		get_data_conf(frame) : get_hole_conf(frame));
+
+	orig_offset = offset;
+	orig_size = count;
+
+	atom_size = get_atom_size(object);
+	atom_bits = get_atom_bits(object);
+
+	/*
+	 * Round-down the start,
+	 * round-up the end.
+	 */
+	resid = offset & (uint64_t)(atom_size - 1);
+
+	if (resid)
+		off_in_head = resid;
+	aligned_offset = offset - off_in_head;
+	expanded_size = orig_size + off_in_head;
+
+	/* calculate tail,
+	   expand size forward  */
+	resid = (offset + orig_size) & (uint64_t)(atom_size - 1);
+
+	if (resid) {
+		off_in_tail = resid;
+		expanded_size += (atom_size - off_in_tail);
+	}
+	/*
+	 * calculate number of occupied blocks
+	 */
+	acount = expanded_size >> atom_bits;
+	/*
+	 * calculate number of full blocks
+	 */
+	size_full_blocks = expanded_size;
+	if (off_in_head)
+		size_full_blocks -= atom_size;
+	if (off_in_tail && size_full_blocks > 0)
+		size_full_blocks -= atom_size;
+	nr_full_blocks = size_full_blocks >> atom_bits;
+
+	conf->atom_size = atom_size;
+	conf->orig_size = orig_size;
+	conf->orig_offset = orig_offset;
+	conf->expanded_size = expanded_size;
+	conf->aligned_offset = aligned_offset;
+
+	conf->off_in_head = off_in_head;
+	conf->off_in_tail = off_in_tail;
+	conf->nr_full_blocks = nr_full_blocks;
+	conf->acount = acount;
+	/*
+	 * Finally, calculate precise amount of
+	 * "extra-bytes" that should be uptodated
+	 * at the end.
+	 * Only if RMW is expected.
+	 */
+	if (off_in_tail && set_gap)
+		set_gap_at_end(frame, object, conf, dtype);
+}
+
+struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE] = {
+	[AES_CIPHER_ALG][XTS_CIPHER_MODE] =
+	{ .atomic = _gf_true,
+	  .should_pad = _gf_true,
+	  .blkbits = AES_BLOCK_BITS,
+	  .init = aes_xts_init,
+	  .set_private = set_private_aes_xts,
+	  .check_key = check_key_aes_xts,
+	  .set_iv = set_iv_aes_xts,
+	  .encrypt = encrypt_aes_xts
+	}
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/keys.c b/xlators/encryption/crypt/src/keys.c
new file mode 100644
index 000000000..4a1d3bb5a
--- /dev/null
+++ b/xlators/encryption/crypt/src/keys.c
@@ -0,0 +1,302 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+/* Key hierarchy
+                  
+                   +----------------+
+                   | MASTER_VOL_KEY |
+                   +-------+--------+
+                           |
+                           |
+          +----------------+----------------+
+          |                |                |
+          |                |                |
+  +-------+------+ +-------+-------+ +------+--------+
+  | NMTD_VOL_KEY | | EMTD_FILE_KEY | | DATA_FILE_KEY |
+  +-------+------+ +---------------+ +---------------+
+          |
+          |
+  +-------+-------+
+  | NMTD_LINK_KEY |
+  +---------------+
+
+ */
+
+#if DEBUG_CRYPT
+static void check_prf_iters(uint32_t num_iters)
+{
+	if (num_iters == 0)
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad number of prf iterations : %d", num_iters);
+}
+#else
+#define check_prf_iters(num_iters) noop
+#endif /* DEBUG_CRYPT */
+
+unsigned char crypt_fake_oid[16] =
+	{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+/*
+ * derive key in the counter mode using
+ * sha256-based HMAC as PRF, see
+ * NIST Special Publication 800-108, 5.1)
+ */
+
+#define PRF_OUTPUT_SIZE  SHA256_DIGEST_LENGTH
+
+static int32_t kderive_init(struct kderive_context *ctx,
+			    const unsigned char *pkey, /* parent key */
+			    uint32_t pkey_size,  /* parent key size */
+			    const unsigned char *idctx, /* id-context */
+			    uint32_t idctx_size,
+			    crypt_key_type type  /* type of child key */)
+{
+	unsigned char *pos;
+	uint32_t llen = strlen(crypt_keys[type].label);	
+	/*
+	 * Compoud the fixed input data for KDF:
+	 * [i]_2 || Label || 0x00 || Id-Context || [L]_2),
+	 * NIST SP 800-108, 5.1
+	 */
+	ctx->fid_len =
+		sizeof(uint32_t) +
+		llen +
+		1 +
+		idctx_size +
+		sizeof(uint32_t);
+
+	ctx->fid = GF_CALLOC(ctx->fid_len, 1, gf_crypt_mt_key);
+	if (!ctx->fid)
+		return ENOMEM;
+	ctx->out_len = round_up(crypt_keys[type].len >> 3,
+				PRF_OUTPUT_SIZE);
+	ctx->out = GF_CALLOC(ctx->out_len, 1, gf_crypt_mt_key);
+	if (!ctx->out) {
+		GF_FREE(ctx->fid);
+		return ENOMEM;
+	}
+	ctx->pkey = pkey;
+	ctx->pkey_len = pkey_size;
+	ctx->ckey_len = crypt_keys[type].len;
+
+	pos = ctx->fid;
+
+	/* counter will be set up in kderive_rfn() */
+	pos += sizeof(uint32_t);
+
+	memcpy(pos, crypt_keys[type].label, llen);
+	pos += llen;
+
+	/* set up zero octet */
+	*pos = 0;
+	pos += 1;
+
+	memcpy(pos, idctx, idctx_size);
+	pos += idctx_size;
+
+	*((uint32_t *)pos) = htobe32(ctx->ckey_len);
+
+	return 0;
+}
+
+static void kderive_update(struct kderive_context *ctx)
+{	
+	uint32_t i;
+	HMAC_CTX hctx;
+	unsigned char *pos = ctx->out;
+	uint32_t *p_iter = (uint32_t *)ctx->fid;
+	uint32_t num_iters = ctx->out_len / PRF_OUTPUT_SIZE;
+
+	check_prf_iters(num_iters);
+
+	HMAC_CTX_init(&hctx);
+	for (i = 0; i < num_iters; i++) {
+		/*
+		 * update the iteration number in the fid
+		 */
+		*p_iter = htobe32(i);
+		HMAC_Init_ex(&hctx,
+			     ctx->pkey, ctx->pkey_len >> 3,
+			     EVP_sha256(),
+			     NULL);
+		HMAC_Update(&hctx, ctx->fid, ctx->fid_len);
+		HMAC_Final(&hctx, pos, NULL);
+
+		pos += PRF_OUTPUT_SIZE;
+	}
+	HMAC_CTX_cleanup(&hctx);
+}
+
+static void kderive_final(struct kderive_context *ctx, unsigned char *child)
+{
+	memcpy(child, ctx->out, ctx->ckey_len >> 3);
+	GF_FREE(ctx->fid);
+	GF_FREE(ctx->out);
+	memset(ctx, 0, sizeof(*ctx));
+}
+
+/*
+ * derive per-volume key for object ids aithentication
+ */
+int32_t get_nmtd_vol_key(struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+
+	ret = kderive_init(&ctx,
+			   master->m_key,
+			   master_key_size(),
+			   crypt_fake_oid, sizeof(uuid_t), NMTD_VOL_KEY);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, master->m_nmtd_key);
+	return 0;
+}
+
+/*
+ * derive per-link key for aithentication of non-encrypted
+ * meta-data (nmtd)
+ */
+int32_t get_nmtd_link_key(loc_t *loc,
+			  struct master_cipher_info *master,
+			  unsigned char *result)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+
+	ret = kderive_init(&ctx,
+			   master->m_nmtd_key,
+			   nmtd_vol_key_size(),
+			   (const unsigned char *)loc->path,
+			   strlen(loc->path), NMTD_LINK_KEY);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, result);
+	return 0;
+}
+
+/*
+ * derive per-file key for encryption and authentication
+ * of encrypted part of metadata (emtd)
+ */
+int32_t get_emtd_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  unsigned char *result)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+
+	ret = kderive_init(&ctx,
+			   master->m_key,
+			   master_key_size(),
+			   info->oid, sizeof(uuid_t), EMTD_FILE_KEY);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, result);
+	return 0;
+}
+
+static int32_t data_key_type_by_size(uint32_t keysize, crypt_key_type *type)
+{
+	int32_t ret = 0;
+	switch (keysize) {
+	case 256:
+		*type = DATA_FILE_KEY_256;
+		break;
+	case 512:
+		*type = DATA_FILE_KEY_512;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Unsupported data key size %d",
+		       keysize);
+		ret = ENOTSUP;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * derive per-file key for data encryption
+ */
+int32_t get_data_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  uint32_t keysize,
+			  unsigned char *key)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+	crypt_key_type type;
+
+	ret = data_key_type_by_size(keysize, &type);
+	if (ret)
+		return ret;
+	ret = kderive_init(&ctx,
+			   master->m_key,
+			   master_key_size(),
+			   info->oid, sizeof(uuid_t), type);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, key);
+	return 0;
+}
+
+/*
+ * NOTE: Don't change existing keys: it will break compatibility;
+ */
+struct crypt_key crypt_keys[LAST_KEY_TYPE] = {
+	[MASTER_VOL_KEY] =
+	{ .len = MASTER_VOL_KEY_SIZE << 3,
+	  .label = "volume-master",
+	},
+	[NMTD_VOL_KEY] =
+	{ .len = NMTD_VOL_KEY_SIZE << 3,
+	  .label = "volume-nmtd-key-generation"
+	},
+	[NMTD_LINK_KEY] =
+	{ .len = 128,
+	  .label = "link-nmtd-authentication"
+	},
+	[EMTD_FILE_KEY] =
+	{ .len = 128,
+	  .label = "file-emtd-encryption-and-auth"
+	},
+	[DATA_FILE_KEY_256] =
+	{ .len = 256,
+	  .label = "file-data-encryption-256"
+	},
+	[DATA_FILE_KEY_512] =
+	{ .len = 512,
+	  .label = "file-data-encryption-512"
+	}
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/metadata.c b/xlators/encryption/crypt/src/metadata.c
new file mode 100644
index 000000000..36b14c055
--- /dev/null
+++ b/xlators/encryption/crypt/src/metadata.c
@@ -0,0 +1,605 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+#include "metadata.h"
+
+int32_t alloc_format(crypt_local_t *local, size_t size)
+{
+	if (size > 0) {
+		local->format = GF_CALLOC(1, size, gf_crypt_mt_mtd);
+		if (!local->format)
+			return ENOMEM;
+	}
+	local->format_size = size;
+	return 0;
+}
+
+int32_t alloc_format_create(crypt_local_t *local)
+{
+	return alloc_format(local, new_format_size());
+}
+
+void free_format(crypt_local_t *local)
+{
+	GF_FREE(local->format);
+}
+
+/*
+ * Check compatibility with extracted metadata
+ */
+static int32_t check_file_metadata(struct crypt_inode_info *info)
+{
+	struct object_cipher_info *object = &info->cinfo;
+
+	if (info->nr_minor != CRYPT_XLATOR_ID) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "unsupported minor subversion %d", info->nr_minor);
+		return EINVAL;
+	}
+	if (object->o_alg > LAST_CIPHER_ALG) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "unsupported cipher algorithm %d",
+		       object->o_alg);
+		return EINVAL;
+	}
+	if (object->o_mode > LAST_CIPHER_MODE) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "unsupported cipher mode %d",
+		       object->o_mode);
+		return EINVAL;
+	}
+	if (object->o_block_bits < CRYPT_MIN_BLOCK_BITS ||
+	    object->o_block_bits > CRYPT_MAX_BLOCK_BITS) {
+		gf_log("crypt", GF_LOG_WARNING, "unsupported block bits %d",
+		       object->o_block_bits);
+		return EINVAL;
+	}
+	/* TBD: check data key size */
+	return 0;
+}
+
+static size_t format_size_v1(mtd_op_t op, size_t old_size)
+{
+	
+	switch (op) {
+	case MTD_CREATE:
+		return sizeof(struct mtd_format_v1);
+	case MTD_OVERWRITE:
+		return old_size;
+	case MTD_APPEND:
+		return old_size + NMTD_8_MAC_SIZE;
+	case MTD_CUT:
+		if (old_size > sizeof(struct mtd_format_v1))
+			return old_size - NMTD_8_MAC_SIZE;
+		else
+			return 0;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad mtd operation");
+		return 0;
+	}
+}
+
+/*
+ * Calculate size of the updated format string.
+ * Returned zero means that we don't need to update the format string.
+ */
+size_t format_size(mtd_op_t op, size_t old_size)
+{
+	size_t versioned;
+
+	versioned = mtd_loaders[current_mtd_loader()].format_size(op,
+				 old_size - sizeof(struct crypt_format));
+	if (versioned != 0)
+		return versioned + sizeof(struct crypt_format);
+	return 0;
+}
+
+/*
+ * size of the format string of newly created file (nr_links = 1)
+ */
+size_t new_format_size(void)
+{
+	return format_size(MTD_CREATE, 0);
+}
+
+/*
+ * Calculate per-link MAC by pathname
+ */
+static int32_t calc_link_mac_v1(struct mtd_format_v1 *fmt,
+				loc_t *loc,
+				unsigned char *result,
+				struct crypt_inode_info *info,
+				struct master_cipher_info *master)
+{
+	int32_t ret;	
+	unsigned char nmtd_link_key[16];
+	CMAC_CTX *cctx;
+	size_t len;
+
+	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not get nmtd link key");
+		return -1;
+	}
+	cctx = CMAC_CTX_new();
+	if (!cctx) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_CTX_new failed");
+		return -1; 
+	}
+	ret = CMAC_Init(cctx, nmtd_link_key, sizeof(nmtd_link_key),
+			EVP_aes_128_cbc(), 0);
+	if (!ret) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_Init failed");
+		CMAC_CTX_free(cctx);
+		return -1;
+	}
+	ret = CMAC_Update(cctx, get_NMTD_V1(info), SIZE_OF_NMTD_V1);
+	if (!ret) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_Update failed");
+		CMAC_CTX_free(cctx);
+		return -1;
+	}
+	ret = CMAC_Final(cctx, result, &len);
+	CMAC_CTX_free(cctx);
+	if (!ret) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_Final failed");
+		return -1;
+	}
+	return 0;
+}
+
+/*
+ * Create per-link MAC of index @idx by pathname
+ */
+static int32_t create_link_mac_v1(struct mtd_format_v1 *fmt,
+				  uint32_t idx,
+				  loc_t *loc,
+				  struct crypt_inode_info *info,
+				  struct master_cipher_info *master)
+{
+	int32_t ret;
+	unsigned char *mac;
+	unsigned char cmac[16];
+
+	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
+
+	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
+	if (ret)
+		return -1;
+	memcpy(mac, cmac, SIZE_OF_NMTD_V1_MAC);
+	return 0;
+}
+
+static int32_t create_format_v1(unsigned char *wire,
+				loc_t *loc,
+				struct crypt_inode_info *info,
+				struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct mtd_format_v1 *fmt;
+	unsigned char mtd_key[16];
+	AES_KEY EMTD_KEY;
+	unsigned char nmtd_link_key[16];
+	uint32_t ad;
+	GCM128_CONTEXT *gctx;
+
+	fmt = (struct mtd_format_v1 *)wire;
+
+	fmt->minor_id = info->nr_minor;
+	fmt->alg_id = AES_CIPHER_ALG;
+	fmt->dkey_factor = master->m_dkey_size >> KEY_FACTOR_BITS;
+	fmt->block_bits = master->m_block_bits;
+	fmt->mode_id = master->m_mode;
+	/*
+	 * retrieve keys for the parts of metadata
+	 */
+	ret = get_emtd_file_key(info, master, mtd_key);
+	if (ret)
+		return ret;
+	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
+	if (ret)
+		return ret;
+	
+	AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
+
+	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
+
+	/* TBD: Check return values */
+
+	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
+
+	ad = htole32(MTD_LOADER_V1);
+	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;
+	}
+	ret = CRYPTO_gcm128_encrypt(gctx,
+				    get_EMTD_V1(fmt),
+				    get_EMTD_V1(fmt),
+				    SIZE_OF_EMTD_V1);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_encrypt failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;
+	}
+	/*
+	 * set MAC of encrypted part of metadata
+	 */
+	CRYPTO_gcm128_tag(gctx, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC);
+	CRYPTO_gcm128_release(gctx);
+	/*
+	 * set the first MAC of non-encrypted part of metadata 
+	 */
+	return create_link_mac_v1(fmt, 0, loc, info, master);
+}
+
+/*
+ * Called by fops:
+ * ->create();
+ * ->link();
+ *
+ * Pack common and version-specific parts of file's metadata
+ * Pre-conditions: @info contains valid object-id.
+ */
+int32_t create_format(unsigned char *wire,
+		      loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master)
+{
+	struct crypt_format *fmt = (struct crypt_format *)wire;
+
+	fmt->loader_id = current_mtd_loader();
+
+	wire += sizeof(struct crypt_format);
+	return mtd_loaders[current_mtd_loader()].create_format(wire, loc,
+							       info, master);
+}
+
+/*
+ * Append or overwrite per-link mac of @mac_idx index
+ * in accordance with the new pathname
+ */
+int32_t appov_link_mac_v1(unsigned char *new,
+			  unsigned char *old,
+			  uint32_t old_size,
+			  int32_t mac_idx,
+			  loc_t *loc,
+			  struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  crypt_local_t *local)
+{
+	memcpy(new, old, old_size);
+	return create_link_mac_v1((struct mtd_format_v1 *)new, mac_idx,
+				  loc, info, master);
+}
+
+/*
+ * Cut per-link mac of @mac_idx index
+ */
+static int32_t cut_link_mac_v1(unsigned char *new,
+			       unsigned char *old,			       
+			       uint32_t old_size,
+			       int32_t mac_idx,
+			       loc_t *loc,
+			       struct crypt_inode_info *info,
+			       struct master_cipher_info *master,
+			       crypt_local_t *local)
+{
+	memcpy(new,
+	       old,
+	       sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1));
+
+	memcpy(new + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1),
+	       old + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx,
+	       old_size - (sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx));
+	return 0;
+}
+
+int32_t update_format_v1(unsigned char *new,
+			 unsigned char *old,
+				size_t old_len,
+				int32_t mac_idx, /* of old name */
+				mtd_op_t op,
+				loc_t *loc,
+				struct crypt_inode_info *info,
+				struct master_cipher_info *master,
+				crypt_local_t *local)
+{
+	switch (op) {
+	case MTD_APPEND:
+		mac_idx = 1 + (old_len - sizeof(struct mtd_format_v1))/8;
+	case MTD_OVERWRITE:
+		return appov_link_mac_v1(new, old, old_len, mac_idx,
+					 loc, info, master, local);
+	case MTD_CUT:
+		return cut_link_mac_v1(new, old, old_len, mac_idx,
+				       loc, info, master, local);
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Bad  mtd operation %d", op);
+		return -1;
+	}
+}
+
+/*
+ * Called by fops:
+ *
+ * ->link()
+ * ->unlink()
+ * ->rename()
+ *
+ */
+int32_t update_format(unsigned char *new,
+		      unsigned char *old,
+		      size_t old_len,
+		      int32_t mac_idx,
+		      mtd_op_t op,
+		      loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master,
+		      crypt_local_t *local)
+{
+	if (!new)
+		return 0;
+	memcpy(new, old, sizeof(struct crypt_format));
+
+	old += sizeof(struct crypt_format);
+	new += sizeof(struct crypt_format);
+	old_len -= sizeof(struct crypt_format);
+
+	return mtd_loaders[current_mtd_loader()].update_format(new, old,
+							       old_len,
+							       mac_idx, op,
+							       loc, info,
+							       master, local);
+}
+
+/*
+ * Perform preliminary checks of found metadata
+ * Return < 0 on errors;
+ * Return number of object-id MACs (>= 1) on success
+ */
+int32_t check_format_v1(uint32_t len, unsigned char *wire)
+{
+	uint32_t nr_links;
+
+	if (len < sizeof(struct mtd_format_v1)) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "v1-loader: bad metadata size %d", len);
+		goto error;
+	}
+	len -= sizeof(struct mtd_format_v1);
+	if (len % sizeof(nmtd_8_mac_t)) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "v1-loader: bad metadata format");
+		goto error;
+	}
+	nr_links = 1 + len / sizeof(nmtd_8_mac_t);
+	if (nr_links > _POSIX_LINK_MAX)
+		goto error;
+	return nr_links;
+ error:
+	return EIO;
+}
+
+/*
+ * Verify per-link MAC specified by index @idx
+ *
+ * return:
+ * -1 on errors;
+ *  0 on failed verification;
+ *  1 on sucessful verification
+ */
+static int32_t verify_link_mac_v1(struct mtd_format_v1 *fmt,
+				  uint32_t idx /* index of the mac to verify */,
+				  loc_t *loc,
+				  struct crypt_inode_info *info,
+				  struct master_cipher_info *master)
+{
+	int32_t ret;
+	unsigned char *mac;
+	unsigned char cmac[16];
+
+	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
+
+	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
+	if (ret)
+		return -1;
+	if (memcmp(cmac, mac, SIZE_OF_NMTD_V1_MAC))
+		return 0;
+	return 1;
+}
+
+/*
+ * Lookup per-link MAC by pathname.
+ *
+ * return index of the MAC, if it was found;
+ * return < 0 on errors, or if the MAC wasn't found
+ */
+static int32_t lookup_link_mac_v1(struct mtd_format_v1 *fmt,
+				  uint32_t nr_macs,
+				  loc_t *loc,
+				  struct crypt_inode_info *info,
+				  struct master_cipher_info *master)
+{
+	int32_t ret;
+	uint32_t idx;
+
+	for (idx = 0; idx < nr_macs; idx++) {
+		ret = verify_link_mac_v1(fmt, idx, loc, info, master);
+		if (ret < 0)
+			return ret;
+		if (ret > 0)
+			return idx;
+	}
+	return -ENOENT;
+}
+
+/*
+ * Extract version-specific part of metadata
+ */
+static int32_t open_format_v1(unsigned char *wire,
+			      int32_t len,
+			      loc_t *loc,
+			      struct crypt_inode_info *info,
+			      struct master_cipher_info *master,
+			      crypt_local_t *local,
+			      gf_boolean_t load_info)
+{
+	int32_t ret;
+	int32_t num_nmtd_macs;
+	struct mtd_format_v1 *fmt;
+	unsigned char mtd_key[16];
+	AES_KEY EMTD_KEY;
+	GCM128_CONTEXT *gctx;
+	uint32_t ad;
+	emtd_8_mac_t gmac;
+	struct object_cipher_info *object;
+
+	num_nmtd_macs = check_format_v1(len, wire);
+	if (num_nmtd_macs <= 0)
+		return EIO;
+	fmt = (struct mtd_format_v1 *)wire;
+
+	ret = lookup_link_mac_v1(fmt, num_nmtd_macs, loc, info, master);
+	if (ret < 0) {
+		gf_log("crypt", GF_LOG_ERROR, "NMTD verification failed");
+		return EINVAL;
+	}
+	local->mac_idx = ret;
+	if (load_info == _gf_false)
+		/* the case of partial open */
+		return 0;
+
+	object = &info->cinfo;
+
+	ret = get_emtd_file_key(info, master, mtd_key);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not retrieve metadata key");
+		return ret;
+	}
+	/*
+	 * decrypt encrypted meta-data
+	 */
+	ret = AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
+	if (ret < 0) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not set encrypt key");
+		return ret;
+	}
+	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
+	if (!gctx) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not alloc gcm context");
+		return ENOMEM;
+	}
+	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
+
+	ad = htole32(MTD_LOADER_V1);
+	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;	
+	}
+	ret = CRYPTO_gcm128_decrypt(gctx,
+				    get_EMTD_V1(fmt),
+				    get_EMTD_V1(fmt),
+				    SIZE_OF_EMTD_V1);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_decrypt failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;
+	}
+	/*
+	 * verify metadata
+	 */
+	CRYPTO_gcm128_tag(gctx, gmac, sizeof(gmac));
+	CRYPTO_gcm128_release(gctx);
+	if (memcmp(gmac, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC)) {
+		gf_log("crypt", GF_LOG_ERROR, "EMTD verification failed");
+		return EINVAL;
+	}
+	/*
+	 * load verified metadata to the private part of inode
+	 */
+	info->nr_minor = fmt->minor_id;
+
+	object->o_alg = fmt->alg_id;
+	object->o_dkey_size = fmt->dkey_factor << KEY_FACTOR_BITS;
+	object->o_block_bits = fmt->block_bits;
+	object->o_mode = fmt->mode_id;
+
+	return check_file_metadata(info);
+}
+
+/*
+ * perform metadata authentication against @loc->path;
+ * extract crypt-specific attribtes and populate @info
+ * with them (optional)
+ */
+int32_t open_format(unsigned char *str,
+		    int32_t len,
+		    loc_t *loc,
+		    struct crypt_inode_info *info,
+		    struct master_cipher_info *master,
+		    crypt_local_t *local,
+		    gf_boolean_t load_info)
+{
+	struct crypt_format *fmt;
+	if (len < sizeof(*fmt)) {
+		gf_log("crypt", GF_LOG_ERROR, "Bad core format");
+		return EIO;
+	}
+	fmt = (struct crypt_format *)str;
+
+	if (fmt->loader_id >= LAST_MTD_LOADER) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "Unsupported loader id %d", fmt->loader_id);
+		return EINVAL;
+	}
+	str += sizeof(*fmt);
+	len -= sizeof(*fmt);
+
+	return mtd_loaders[fmt->loader_id].open_format(str,
+						       len,
+						       loc,
+						       info,
+						       master,
+						       local,
+						       load_info);
+}
+
+struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER] = {
+	[MTD_LOADER_V1] = 
+	{.format_size = format_size_v1,
+	 .create_format = create_format_v1,
+	 .open_format = open_format_v1,
+	 .update_format = update_format_v1
+	}
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/metadata.h b/xlators/encryption/crypt/src/metadata.h
new file mode 100644
index 000000000..a92f149ef
--- /dev/null
+++ b/xlators/encryption/crypt/src/metadata.h
@@ -0,0 +1,74 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __METADATA_H__
+#define __METADATA_H__
+
+#define NMTD_8_MAC_SIZE   (8)
+#define EMTD_8_MAC_SIZE   (8)
+
+typedef uint8_t nmtd_8_mac_t[NMTD_8_MAC_SIZE];
+typedef uint8_t emtd_8_mac_t[EMTD_8_MAC_SIZE] ;
+
+/*
+ * Version "v1" of file's metadata.
+ * Metadata of this version has 4 components:
+ * 
+ * 1) EMTD (Encrypted part of MeTaData);
+ * 2) NMTD (Non-encrypted part of MeTaData);
+ * 3) EMTD_MAC; (EMTD Message Authentication Code);
+ * 4) Array of per-link NMTD MACs (for every (hard)link it includes
+ *    exactly one MAC)
+ */
+struct mtd_format_v1 {
+	/* EMTD, encrypted part of meta-data */
+	uint8_t            alg_id;  /* cipher algorithm id (only AES for now) */
+        uint8_t            mode_id; /* cipher mode id; (only XTS for now) */
+	uint8_t            block_bits; /* encoded block size */
+	uint8_t            minor_id; /* client translator id */
+	uint8_t            dkey_factor; /* encoded size of the data key */
+	/* MACs */
+	emtd_8_mac_t       gmac; /* MAC of the encrypted meta-data, 8 bytes */
+	nmtd_8_mac_t       omac; /* per-link MACs of the non-encrypted
+				  * meta-data: at least one such MAC is always
+				  * present */
+} __attribute__((packed));
+
+/*
+ * NMTD, the non-encrypted part of metadata of version "v1"
+ * is file's gfid, which is generated on trusted machines.
+ */
+#define SIZE_OF_NMTD_V1 (sizeof(uuid_t))
+#define SIZE_OF_EMTD_V1 (offsetof(struct mtd_format_v1, gmac) -		\
+			 offsetof(struct mtd_format_v1, alg_id))
+#define SIZE_OF_NMTD_V1_MAC (NMTD_8_MAC_SIZE)
+#define SIZE_OF_EMTD_V1_MAC (EMTD_8_MAC_SIZE)
+
+static inline unsigned char *get_EMTD_V1(struct mtd_format_v1 *format)
+{
+        return &format->alg_id;
+}
+
+static inline unsigned char *get_NMTD_V1(struct crypt_inode_info *info)
+{
+        return info->oid;
+}
+
+static inline unsigned char *get_EMTD_V1_MAC(struct mtd_format_v1 *format)
+{
+        return format->gmac;
+}
+
+static inline unsigned char *get_NMTD_V1_MAC(struct mtd_format_v1 *format)
+{
+        return format->omac;
+}
+
+#endif /* __METADATA_H__ */
diff --git a/xlators/features/Makefile.am b/xlators/features/Makefile.am
index 0e37fc0b7..d2f5ef192 100644
--- a/xlators/features/Makefile.am
+++ b/xlators/features/Makefile.am
@@ -1,4 +1,4 @@
 SUBDIRS = locks quota read-only mac-compat quiesce marker index \
-	  protect changelog gfid-access $(GLUPY_SUBDIR) qemu-block # trash path-converter # filter
+	  protect compress changelog gfid-access $(GLUPY_SUBDIR) qemu-block # trash path-converter # filter
 
 CLEANFILES =
diff --git a/xlators/features/changelog/lib/src/gf-changelog.c b/xlators/features/changelog/lib/src/gf-changelog.c
index 15c6e8414..ca8e373e7 100644
--- a/xlators/features/changelog/lib/src/gf-changelog.c
+++ b/xlators/features/changelog/lib/src/gf-changelog.c
@@ -483,8 +483,8 @@ gf_changelog_register (char *brick_path, char *scratch_dir,
                 goto cleanup;
         }
 
-        ret = pthread_create (&gfc->gfc_changelog_processor,
-                              NULL, gf_changelog_process, gfc);
+        ret = gf_thread_create (&gfc->gfc_changelog_processor,
+				NULL, gf_changelog_process, gfc);
         if (ret) {
                 errn = errno;
                 gf_log (this->name, GF_LOG_ERROR,
diff --git a/xlators/features/changelog/src/changelog-encoders.c b/xlators/features/changelog/src/changelog-encoders.c
index c71ea9270..553eec85c 100644
--- a/xlators/features/changelog/src/changelog-encoders.c
+++ b/xlators/features/changelog/src/changelog-encoders.c
@@ -154,3 +154,23 @@ changelog_encode_binary (xlator_t *this, changelog_log_data_t *cld)
 
         return changelog_write_change (priv, buffer, off);
 }
+
+static struct changelog_encoder
+cb_encoder[] = {
+        [CHANGELOG_ENCODE_BINARY] =
+        {
+                .encoder = CHANGELOG_ENCODE_BINARY,
+                .encode = changelog_encode_binary,
+        },
+        [CHANGELOG_ENCODE_ASCII] =
+        {
+                .encoder = CHANGELOG_ENCODE_ASCII,
+                .encode = changelog_encode_ascii,
+        },
+};
+
+void
+changelog_encode_change( changelog_priv_t * priv)
+{
+        priv->ce = &cb_encoder[priv->encode_mode];
+}
diff --git a/xlators/features/changelog/src/changelog-encoders.h b/xlators/features/changelog/src/changelog-encoders.h
index 43deb1307..a3efbee05 100644
--- a/xlators/features/changelog/src/changelog-encoders.h
+++ b/xlators/features/changelog/src/changelog-encoders.h
@@ -40,5 +40,7 @@ int
 changelog_encode_binary (xlator_t *, changelog_log_data_t *);
 int
 changelog_encode_ascii (xlator_t *, changelog_log_data_t *);
+void
+changelog_encode_change(changelog_priv_t *);
 
 #endif /* _CHANGELOG_ENCODERS_H */
diff --git a/xlators/features/changelog/src/changelog-helpers.c b/xlators/features/changelog/src/changelog-helpers.c
index c1bb6e5fe..7ab0091b5 100644
--- a/xlators/features/changelog/src/changelog-helpers.c
+++ b/xlators/features/changelog/src/changelog-helpers.c
@@ -21,6 +21,7 @@
 #include "changelog-helpers.h"
 #include "changelog-mem-types.h"
 
+#include "changelog-encoders.h"
 #include <pthread.h>
 
 void
@@ -201,7 +202,7 @@ changelog_open (xlator_t *this,
         (void) snprintf (buffer, 1024, CHANGELOG_HEADER,
                          CHANGELOG_VERSION_MAJOR,
                          CHANGELOG_VERSION_MINOR,
-                         priv->encode_mode);
+                         priv->ce->encoder);
         ret = changelog_write_change (priv, buffer, strlen (buffer));
         if (ret) {
                 close (priv->changelog_fd);
@@ -267,6 +268,7 @@ changelog_handle_change (xlator_t *this,
         int ret = 0;
 
         if (CHANGELOG_TYPE_IS_ROLLOVER (cld->cld_type)) {
+                changelog_encode_change(priv);
                 ret = changelog_start_next_change (this, priv,
                                                    cld->cld_roll_time,
                                                    cld->cld_finale);
diff --git a/xlators/features/changelog/src/changelog.c b/xlators/features/changelog/src/changelog.c
index 0e85ee7a0..cea0e8c70 100644
--- a/xlators/features/changelog/src/changelog.c
+++ b/xlators/features/changelog/src/changelog.c
@@ -36,20 +36,6 @@ cb_bootstrap[] = {
         },
 };
 
-static struct changelog_encoder
-cb_encoder[] = {
-        [CHANGELOG_ENCODE_BINARY] =
-        {
-                .encoder = CHANGELOG_ENCODE_BINARY,
-                .encode = changelog_encode_binary,
-        },
-        [CHANGELOG_ENCODE_ASCII] =
-        {
-                .encoder = CHANGELOG_ENCODE_ASCII,
-                .encode = changelog_encode_ascii,
-        },
-};
-
 /* Entry operations - TYPE III */
 
 /**
@@ -1030,15 +1016,15 @@ changelog_spawn_helper_threads (xlator_t *this, changelog_priv_t *priv)
         int ret = 0;
 
         priv->cr.this = this;
-        ret = pthread_create (&priv->cr.rollover_th,
-                              NULL, changelog_rollover, priv);
+        ret = gf_thread_create (&priv->cr.rollover_th,
+				NULL, changelog_rollover, priv);
         if (ret)
                 goto out;
 
         if (priv->fsync_interval) {
                 priv->cf.this = this;
-                ret = pthread_create (&priv->cf.fsync_th,
-                                      NULL, changelog_fsync_thread, priv);
+                ret = gf_thread_create (&priv->cf.fsync_th,
+					NULL, changelog_fsync_thread, priv);
         }
 
         if (ret)
@@ -1102,8 +1088,8 @@ changelog_spawn_notifier (xlator_t *this, changelog_priv_t *priv)
         priv->cn.this = this;
         priv->cn.rfd  = pipe_fd[0];
 
-        ret = pthread_create (&priv->cn.notify_th,
-                              NULL, changelog_notifier, priv);
+        ret = gf_thread_create (&priv->cn.notify_th,
+				NULL, changelog_notifier, priv);
 
  out:
         return ret;
@@ -1363,8 +1349,7 @@ init (xlator_t *this)
 
         GF_OPTION_INIT ("fsync-interval", priv->fsync_interval, int32, out);
 
-        GF_ASSERT (cb_encoder[priv->encode_mode].encoder == priv->encode_mode);
-        priv->ce = &cb_encoder[priv->encode_mode];
+        changelog_encode_change(priv);
 
         GF_ASSERT (cb_bootstrap[priv->op_mode].mode == priv->op_mode);
         priv->cb = &cb_bootstrap[priv->op_mode];
diff --git a/xlators/storage/bd_map/Makefile.am b/xlators/features/compress/Makefile.am
index a985f42a8..a985f42a8 100644
--- a/xlators/storage/bd_map/Makefile.am
+++ b/xlators/features/compress/Makefile.am
diff --git a/xlators/features/compress/src/Makefile.am b/xlators/features/compress/src/Makefile.am
new file mode 100644
index 000000000..4a64b52a9
--- /dev/null
+++ b/xlators/features/compress/src/Makefile.am
@@ -0,0 +1,17 @@
+xlator_LTLIBRARIES = cdc.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+noinst_HEADERS = cdc.h cdc-mem-types.h
+
+cdc_la_LDFLAGS = -module -avoidversion $(LIBZ_LIBS)
+
+cdc_la_SOURCES = cdc.c cdc-helper.c
+cdc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) \
+-shared -nostartfiles  $(LIBZ_CFLAGS)
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/compress/src/cdc-helper.c b/xlators/features/compress/src/cdc-helper.c
new file mode 100644
index 000000000..54432ff45
--- /dev/null
+++ b/xlators/features/compress/src/cdc-helper.c
@@ -0,0 +1,547 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+
+#include "cdc.h"
+#include "cdc-mem-types.h"
+
+#ifdef HAVE_LIB_Z
+#include "zlib.h"
+#endif
+
+#ifdef HAVE_LIB_Z
+/* gzip header looks something like this
+ * (RFC 1950)
+ *
+ * +---+---+---+---+---+---+---+---+---+---+
+ * |ID1|ID2|CM |FLG|     MTIME     |XFL|OS |
+ * +---+---+---+---+---+---+---+---+---+---+
+ *
+ * Data is usually sent without this header i.e
+ * Data sent = <compressed-data> + trailer(8)
+ * The trailer contains the checksum.
+ *
+ * gzip_header is added only during debugging.
+ * Refer to the function cdc_dump_iovec_to_disk
+ */
+static const char gzip_header[10] =
+        {
+                '\037', '\213', Z_DEFLATED, 0,
+                0,      0,      0,          0,
+                0,      GF_CDC_OS_ID
+        };
+
+static int32_t
+cdc_next_iovec (xlator_t *this, cdc_info_t *ci)
+{
+        int ret = -1;
+
+        ci->ncount++;
+        /* check for iovec overflow -- should not happen */
+        if (ci->ncount == MAX_IOVEC) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Zlib output buffer overflow"
+                        " ->ncount (%d) | ->MAX_IOVEC (%d)",
+                        ci->ncount, MAX_IOVEC);
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static void
+cdc_put_long (unsigned char *string, unsigned long x)
+{
+        string[0] = (unsigned char) (x & 0xff);
+        string[1] = (unsigned char) ((x & 0xff00) >> 8);
+        string[2] = (unsigned char) ((x & 0xff0000) >> 16);
+        string[3] = (unsigned char) ((x & 0xff000000) >> 24);
+}
+
+static unsigned long
+cdc_get_long (unsigned char *buf)
+{
+        return ((unsigned long) buf[0])
+                | (((unsigned long) buf[1]) << 8)
+                | (((unsigned long) buf[2]) << 16)
+                | (((unsigned long) buf[3]) << 24);
+}
+
+static int32_t
+cdc_init_gzip_trailer (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+{
+        int   ret = -1;
+        char *buf = NULL;
+
+        ret = cdc_next_iovec (this, ci);
+        if (ret)
+                goto out;
+
+        buf = CURR_VEC(ci).iov_base =
+                (char *) GF_CALLOC (1, GF_CDC_VALIDATION_SIZE,
+                                    gf_cdc_mt_gzip_trailer_t);
+
+        if (!CURR_VEC(ci).iov_base)
+                goto out;
+
+        CURR_VEC(ci).iov_len = GF_CDC_VALIDATION_SIZE;
+
+        cdc_put_long ((unsigned char *)&buf[0], ci->crc);
+        cdc_put_long ((unsigned char *)&buf[4], ci->stream.total_in);
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static int32_t
+cdc_alloc_iobuf_and_init_vec (xlator_t *this,
+                              cdc_priv_t *priv, cdc_info_t *ci,
+                              int size)
+{
+        int           ret       = -1;
+        int           alloc_len = 0;
+        struct iobuf *iobuf     = NULL;
+
+        ret = cdc_next_iovec (this, ci);
+        if (ret)
+                goto out;
+
+        alloc_len = size ? size : ci->buffer_size;
+
+        iobuf = iobuf_get2 (this->ctx->iobuf_pool, alloc_len);
+        if (!iobuf)
+                goto out;
+
+        ret = iobref_add (ci->iobref, iobuf);
+        if (ret)
+                goto out;
+
+        /* Initialize this iovec */
+        CURR_VEC(ci).iov_base = iobuf->ptr;
+        CURR_VEC(ci).iov_len  = alloc_len;
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static void
+cdc_init_zlib_output_stream (cdc_priv_t *priv, cdc_info_t *ci, int size)
+{
+        ci->stream.next_out  = (unsigned char *) CURR_VEC(ci).iov_base;
+        ci->stream.avail_out = size ? size : ci->buffer_size;
+}
+
+/* This routine is for testing and debugging only.
+ * Data written = header(10) + <compressed-data> + trailer(8)
+ * So each gzip dump file is at least 18 bytes in size.
+ */
+void
+cdc_dump_iovec_to_disk (xlator_t *this, cdc_info_t *ci, const char *file)
+{
+        int    i      = 0;
+        int    fd     = 0;
+        size_t writen = 0;
+        size_t total_writen = 0;
+
+        fd = open (file, O_WRONLY|O_CREAT|O_TRUNC, 0777 );
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Cannot open file: %s", file);
+                return;
+        }
+
+        writen = write (fd, (char *) gzip_header, 10);
+        total_writen += writen;
+        for (i = 0; i < ci->ncount; i++) {
+                writen = write (fd, (char *) ci->vec[i].iov_base, ci->vec[i].iov_len);
+                total_writen += writen;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                        "dump'd %zu bytes to %s", total_writen, GF_CDC_DEBUG_DUMP_FILE );
+
+        close (fd);
+}
+
+static int32_t
+cdc_flush_libz_buffer (cdc_priv_t *priv, xlator_t *this, cdc_info_t *ci,
+                       int (*libz_func)(z_streamp, int),
+                       int flush)
+{
+        int32_t ret = Z_OK;
+        int done = 0;
+        unsigned int deflate_len = 0;
+
+        for (;;) {
+                deflate_len = ci->buffer_size - ci->stream.avail_out;
+
+                if (deflate_len != 0) {
+                        CURR_VEC(ci).iov_len = deflate_len;
+
+                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+                        if (ret) {
+                                ret = Z_MEM_ERROR;
+                                break;
+                        }
+
+                        /* Re-position Zlib output buffer */
+                        cdc_init_zlib_output_stream (priv, ci, 0);
+                }
+
+                if (done) {
+                        ci->ncount--;
+                        break;
+                }
+
+                ret = libz_func (&ci->stream, flush);
+
+                if (ret == Z_BUF_ERROR) {
+                        ret = Z_OK;
+                        ci->ncount--;
+                        break;
+                }
+
+                done = (ci->stream.avail_out != 0 || ret == Z_STREAM_END);
+
+                if (ret != Z_OK && ret != Z_STREAM_END)
+                        break;
+        }
+
+        return ret;
+}
+
+static int32_t
+do_cdc_compress (struct iovec *vec, xlator_t *this, cdc_priv_t *priv,
+                 cdc_info_t *ci)
+{
+        int ret = -1;
+
+        /* Initialize defalte */
+        ret = deflateInit2 (&ci->stream, priv->cdc_level, Z_DEFLATED,
+                            priv->window_size, priv->mem_level,
+                            Z_DEFAULT_STRATEGY);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "unable to init Zlib (retval: %d)", ret);
+                goto out;
+        }
+
+        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+        if (ret)
+                goto out;
+
+        /* setup output buffer */
+        cdc_init_zlib_output_stream (priv, ci, 0);
+
+        /* setup input buffer */
+        ci->stream.next_in  = (unsigned char *) vec->iov_base;
+        ci->stream.avail_in = vec->iov_len;
+
+        ci->crc = crc32 (ci->crc, (const Bytef *) vec->iov_base, vec->iov_len);
+
+        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%d buffer_size=%d",
+                ci->crc, ci->stream.avail_in, ci->buffer_size);
+
+        /* compress !! */
+        while (ci->stream.avail_in != 0) {
+                if (ci->stream.avail_out == 0) {
+
+                        CURR_VEC(ci).iov_len = ci->buffer_size;
+
+                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+                        if (ret)
+                                break;
+
+                        /* Re-position Zlib output buffer */
+                        cdc_init_zlib_output_stream (priv, ci, 0);
+                }
+
+                ret = deflate (&ci->stream, Z_NO_FLUSH);
+                if (ret != Z_OK)
+                        break;
+        }
+
+ out:
+        return ret;
+}
+
+int32_t
+cdc_compress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
+              dict_t **xdata)
+{
+        int ret = -1;
+        int i   = 0;
+
+        ci->iobref = iobref_new ();
+        if (!ci->iobref)
+                goto out;
+
+        if (!*xdata) {
+                *xdata = dict_new ();
+                if (!*xdata) {
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot allocate xdata"
+                                " dict");
+                        goto out;
+                }
+        }
+
+        /* data */
+        for (i = 0; i < ci->count; i++) {
+                ret = do_cdc_compress (&ci->vector[i], this, priv, ci);
+                if (ret != Z_OK)
+                        goto deflate_cleanup_out;
+        }
+
+        /* flush zlib buffer */
+        ret = cdc_flush_libz_buffer (priv, this, ci, deflate, Z_FINISH);
+        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Compression Error: ret (%d)", ret);
+                ret = -1;
+                goto deflate_cleanup_out;
+        }
+
+        /* trailer */
+        ret = cdc_init_gzip_trailer (this, priv, ci);
+        if (ret)
+                goto deflate_cleanup_out;
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Compressed %ld to %ld bytes",
+                ci->stream.total_in, ci->stream.total_out);
+
+        ci->nbytes = ci->stream.total_out + GF_CDC_VALIDATION_SIZE;
+
+        /* set deflated canary value for identification */
+        ret = dict_set_int32 (*xdata, GF_CDC_DEFLATE_CANARY_VAL, 1);
+        if (ret) {
+                /* Send uncompressed data if we can't _tell_ the client
+                 * that deflated data is on it's way. So, we just log
+                 * the faliure and continue as usual.
+                 */
+                 gf_log (this->name, GF_LOG_ERROR,
+                 "Data deflated, but could not set canary"
+                 " value in dict for identification");
+        }
+
+        /* This is to be used in testing */
+        if ( priv->debug ) {
+                cdc_dump_iovec_to_disk (this, ci, GF_CDC_DEBUG_DUMP_FILE );
+        }
+
+ deflate_cleanup_out:
+        (void) deflateEnd(&ci->stream);
+
+ out:
+        return ret;
+}
+
+
+/* deflate content is checked by the presence of a canary
+ * value in the dict as the key
+ */
+static int32_t
+cdc_check_content_for_deflate (dict_t *xdata)
+{
+        return dict_get (xdata, GF_CDC_DEFLATE_CANARY_VAL) ? -1 : 0;
+}
+
+static unsigned long
+cdc_extract_crc (char *trailer)
+{
+        return cdc_get_long ((unsigned char *) &trailer[0]);
+}
+
+static unsigned long
+cdc_extract_size (char *trailer)
+{
+        return cdc_get_long ((unsigned char *) &trailer[4]);
+}
+
+static int32_t
+cdc_validate_inflate (cdc_info_t *ci, unsigned long crc,
+                      unsigned long len)
+{
+        return !((crc == ci->crc)
+                 /* inflated length is hidden inside
+                  * Zlib stream struct */
+                 && (len == ci->stream.total_out));
+}
+
+static int32_t
+do_cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+{
+        int            ret          = -1;
+        int            i            = 0;
+        int            len          = 0;
+        char          *inflte       = NULL;
+        char          *trailer      = NULL;
+        struct iovec   vec          = {0,};
+        unsigned long  computed_crc = 0;
+        unsigned long  computed_len = 0;
+
+        ret = inflateInit2 (&ci->stream, priv->window_size);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Zlib: Unable to initialize inflate");
+                goto out;
+        }
+
+        vec = THIS_VEC(ci, 0);
+
+        trailer = (char *) (((char *) vec.iov_base) + vec.iov_len
+                - GF_CDC_VALIDATION_SIZE);
+
+        /* CRC of uncompressed data */
+        computed_crc = cdc_extract_crc (trailer);
+
+        /* size of uncomrpessed data */
+        computed_len = cdc_extract_size (trailer);
+
+        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%lu buffer_size=%d",
+                computed_crc, computed_len, ci->buffer_size);
+
+        inflte = vec.iov_base ;
+        len = vec.iov_len - GF_CDC_VALIDATION_SIZE;
+
+        /* allocate buffer of the original length of the data */
+        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+        if (ret)
+                goto out;
+
+        /* setup output buffer */
+        cdc_init_zlib_output_stream (priv, ci, 0);
+
+        /* setup input buffer */
+        ci->stream.next_in = (unsigned char *) inflte;
+        ci->stream.avail_in = len;
+
+        while (ci->stream.avail_in != 0) {
+                if (ci->stream.avail_out == 0) {
+                        CURR_VEC(ci).iov_len = ci->buffer_size;
+
+                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+                        if (ret)
+                                break;
+
+                        /* Re-position Zlib output buffer */
+                        cdc_init_zlib_output_stream (priv, ci, 0);
+                }
+
+                ret = inflate (&ci->stream, Z_NO_FLUSH);
+                if (ret == Z_STREAM_ERROR)
+                        break;
+        }
+
+        /* flush zlib buffer */
+        ret = cdc_flush_libz_buffer (priv, this, ci, inflate, Z_SYNC_FLUSH);
+        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Decompression Error: ret (%d)", ret);
+                ret = -1;
+                goto out;
+        }
+
+        /* compute CRC of the uncompresses data to check for
+         * correctness */
+
+        for (i = 0; i < ci->ncount; i++) {
+                ci->crc = crc32 (ci->crc,
+                                 (const Bytef *) ci->vec[i].iov_base,
+                                 ci->vec[i].iov_len);
+        }
+
+        /* validate inflated data */
+        ret = cdc_validate_inflate (ci, computed_crc, computed_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Checksum or length mismatched in inflated data");
+        }
+
+ out:
+        return ret;
+}
+
+int32_t
+cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
+                dict_t *xdata)
+{
+        int32_t ret = -1;
+
+        /* check for deflate content */
+        if (!cdc_check_content_for_deflate (xdata)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Content not deflated, passing through ...");
+                goto passthrough_out;
+        }
+
+        ci->iobref = iobref_new ();
+        if (!ci->iobref)
+                goto passthrough_out;
+
+        /* do we need to do this? can we assume that one iovec
+         * will hold per request data everytime?
+         *
+         * server/client protocol seems to deal with a single
+         * iovec even if op_ret > 1M. So, it looks ok to
+         * assume that a single iovec will contain all the
+         * data (This saves us a lot from finding the trailer
+         * and the data since it could have been split-up onto
+         * two adjacent iovec's.
+         *
+         * But, in case this translator is loaded above quick-read
+         * for some reason, then it's entirely possible that we get
+         * multiple iovec's...
+         *
+         * This case (handled below) is not tested. (by loading the
+         * xlator below quick-read)
+         */
+
+        /* @@ I_HOPE_THIS_IS_NEVER_HIT */
+        if (ci->count > 1) {
+                gf_log (this->name, GF_LOG_WARNING, "unable to handle"
+                        " multiple iovecs (%d in number)", ci->count);
+                goto inflate_cleanup_out;
+                /* TODO: coallate all iovecs in one */
+        }
+
+        ret = do_cdc_decompress (this, priv, ci);
+        if (ret)
+                goto inflate_cleanup_out;
+
+        ci->nbytes = ci->stream.total_out;
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Inflated %ld to %ld bytes",
+                ci->stream.total_in, ci->stream.total_out);
+
+ inflate_cleanup_out:
+        (void) inflateEnd (&ci->stream);
+
+ passthrough_out:
+        return ret;
+}
+
+#endif
diff --git a/xlators/features/compress/src/cdc-mem-types.h b/xlators/features/compress/src/cdc-mem-types.h
new file mode 100644
index 000000000..efa008059
--- /dev/null
+++ b/xlators/features/compress/src/cdc-mem-types.h
@@ -0,0 +1,22 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CDC_MEM_TYPES_H
+#define __CDC_MEM_TYPES_H
+
+#include "mem-types.h"
+
+enum gf_cdc_mem_types {
+        gf_cdc_mt_priv_t         = gf_common_mt_end + 1,
+        gf_cdc_mt_vec_t          = gf_common_mt_end + 2,
+        gf_cdc_mt_gzip_trailer_t = gf_common_mt_end + 3,
+};
+
+#endif
diff --git a/xlators/features/compress/src/cdc.c b/xlators/features/compress/src/cdc.c
new file mode 100644
index 000000000..eb7d87c56
--- /dev/null
+++ b/xlators/features/compress/src/cdc.c
@@ -0,0 +1,342 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <sys/uio.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+
+#include "cdc.h"
+#include "cdc-mem-types.h"
+
+static void
+cdc_cleanup_iobref (cdc_info_t *ci)
+{
+        assert(ci->iobref != NULL);
+        iobref_clear (ci->iobref);
+}
+
+int32_t
+cdc_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               struct iovec *vector, int32_t count,
+               struct iatt *stbuf, struct iobref *iobref,
+               dict_t *xdata)
+{
+        int         ret   = -1;
+        cdc_priv_t *priv  = NULL;
+        cdc_info_t  ci    = {0,};
+
+        GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
+        GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
+
+        priv = this->private;
+
+        if (op_ret <= 0)
+                goto default_out;
+
+        if ( (priv->min_file_size != 0)
+             && (op_ret < priv->min_file_size) )
+                goto default_out;
+
+        ci.count       = count;
+        ci.ibytes      = op_ret;
+        ci.vector      = vector;
+        ci.buf         = NULL;
+        ci.iobref      = NULL;
+        ci.ncount      = 0;
+        ci.crc         = 0;
+        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+/* A readv compresses on the server side and decompresses on the client side
+ */
+        if (priv->op_mode == GF_CDC_MODE_SERVER) {
+                ret = cdc_compress (this, priv, &ci, &xdata);
+        } else if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+                ret = cdc_decompress (this, priv, &ci, xdata);
+        } else {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid operation mode (%d)", priv->op_mode);
+        }
+
+        if (ret)
+                goto default_out;
+
+        STACK_UNWIND_STRICT (readv, frame, ci.nbytes, op_errno,
+                             ci.vec, ci.ncount, stbuf, iobref,
+                             xdata);
+        cdc_cleanup_iobref (&ci);
+        return 0;
+
+ default_out:
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
+                             vector, count, stbuf, iobref, xdata);
+        return 0;
+}
+
+int32_t
+cdc_readv (call_frame_t *frame, xlator_t *this,
+           fd_t *fd, size_t size, off_t offset, uint32_t flags,
+           dict_t *xdata)
+{
+        fop_readv_cbk_t cbk = NULL;
+
+#ifdef HAVE_LIB_Z
+        cbk = cdc_readv_cbk;
+#else
+        cbk = default_readv_cbk;
+#endif
+        STACK_WIND (frame, cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+        return 0;
+}
+
+int32_t
+cdc_writev_cbk (call_frame_t *frame,
+                void *cookie,
+                xlator_t *this,
+                int32_t op_ret,
+                int32_t op_errno,
+                struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+
+	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+	return 0;
+}
+
+int32_t
+cdc_writev (call_frame_t *frame,
+            xlator_t *this,
+            fd_t *fd,
+            struct iovec *vector,
+            int32_t count,
+            off_t offset,
+            uint32_t flags,
+            struct iobref *iobref, dict_t *xdata)
+{
+	int	     ret   = -1;
+	cdc_priv_t  *priv  = NULL;
+	cdc_info_t   ci    = {0,};
+	size_t       isize = 0;
+
+	GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
+	GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
+
+	priv = this->private;
+
+	isize =  iov_length(vector, count);
+
+	if (isize <= 0)
+                goto default_out;
+
+        if ( (priv->min_file_size != 0)
+             && (isize < priv->min_file_size) )
+                goto default_out;
+
+        ci.count       = count;
+        ci.ibytes      = isize;
+        ci.vector      = vector;
+        ci.buf         = NULL;
+        ci.iobref      = NULL;
+        ci.ncount      = 0;
+        ci.crc         = 0;
+        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+/* A writev compresses on the client side and decompresses on the server side
+ */
+	if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+		ret = cdc_compress (this, priv, &ci, &xdata);
+	} else if (priv->op_mode == GF_CDC_MODE_SERVER) {
+		ret = cdc_decompress (this, priv, &ci, xdata);
+	} else {
+		gf_log (this->name, GF_LOG_ERROR, "Invalid operation mode (%d) ", priv->op_mode);
+	}
+
+	if (ret)
+		goto default_out;
+
+	STACK_WIND (frame,
+                    cdc_writev_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->writev,
+                    fd, ci.vec, ci.ncount, offset, flags,
+                    iobref, xdata);
+
+        cdc_cleanup_iobref (&ci);
+        return 0;
+
+ default_out:
+	STACK_WIND (frame,
+                    cdc_writev_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->writev,
+                    fd, vector, count, offset, flags,
+                    iobref, xdata);
+	return 0;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        int         ret      = -1;
+        char       *temp_str = NULL;
+        cdc_priv_t *priv     = NULL;
+
+        GF_VALIDATE_OR_GOTO ("cdc", this, err);
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Need subvolume == 1");
+                goto err;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Dangling volume. Check volfile");
+        }
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_cdc_mt_priv_t);
+        if (!priv) {
+                goto err;
+        }
+
+        /* Check if debug mode is turned on */
+        GF_OPTION_INIT ("debug", priv->debug, bool, err);
+        if( priv->debug ) {
+                gf_log (this->name, GF_LOG_DEBUG, "CDC debug option turned on");
+        }
+
+        /* Set Gzip Window Size */
+        GF_OPTION_INIT ("window-size", priv->window_size, int32, err);
+        if ( (priv->window_size > GF_CDC_MAX_WINDOWSIZE)
+             || (priv->window_size < GF_CDC_DEF_WINDOWSIZE) ) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Invalid gzip window size (%d), using default",
+                        priv->window_size);
+                priv->window_size = GF_CDC_DEF_WINDOWSIZE;
+        }
+
+        /* Set Gzip (De)Compression Level */
+        GF_OPTION_INIT ("compression-level", priv->cdc_level, int32, err);
+        if ( ((priv->cdc_level < 1) || (priv->cdc_level > 9))
+             && (priv->cdc_level != GF_CDC_DEF_COMPRESSION) ) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Invalid gzip (de)compression level (%d),"
+                        " using default", priv->cdc_level);
+                priv->cdc_level = GF_CDC_DEF_COMPRESSION;
+        }
+
+        /* Set Gzip Memory Level */
+        GF_OPTION_INIT ("mem-level", priv->mem_level, int32, err);
+        if ( (priv->mem_level < 1) || (priv->mem_level > 9) ) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Invalid gzip memory level, using the default");
+                priv->mem_level = GF_CDC_DEF_MEMLEVEL;
+        }
+
+        /* Set min file size to enable compression */
+        GF_OPTION_INIT ("min-size", priv->min_file_size, int32, err);
+
+        /* Mode of operation - Server/Client */
+        ret = dict_get_str (this->options, "mode", &temp_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Operation mode not specified !!");
+                goto err;
+        }
+
+        if (GF_CDC_MODE_IS_CLIENT (temp_str)) {
+                priv->op_mode = GF_CDC_MODE_CLIENT;
+        } else if (GF_CDC_MODE_IS_SERVER (temp_str)) {
+                priv->op_mode = GF_CDC_MODE_SERVER;
+        } else {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Bogus operation mode (%s) specified", temp_str);
+                goto err;
+        }
+
+        this->private = priv;
+        gf_log (this->name, GF_LOG_DEBUG, "CDC xlator loaded in (%s) mode",temp_str);
+        return 0;
+
+ err:
+        if (priv)
+                GF_FREE (priv);
+
+        return -1;
+}
+
+void
+fini (xlator_t *this)
+{
+        cdc_priv_t *priv = this->private;
+
+        if (priv)
+                GF_FREE (priv);
+        this->private = NULL;
+        return;
+}
+
+struct xlator_fops fops = {
+        .readv  = cdc_readv,
+        .writev = cdc_writev,
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+        { .key  = {"window-size"},
+          .default_value = "-15",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Size of the zlib history buffer."
+        },
+        { .key  = {"mem-level"},
+          .default_value = "8",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Memory allocated for internal compression state.\
+                          1 uses minimum memory but is slow and reduces \
+                          compression ratio; memLevel=9 uses maximum memory \
+                          for optimal speed. The default value is 8."
+        },
+        { .key  = {"compression-level"},
+          .default_value = "-1",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Compression levels \
+                          0 : no compression, 1 : best speed, \
+                          9 : best compression, -1 : default compression "
+        },
+        { .key  = {"min-size"},
+          .default_value = "0",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Data is compressed only when its size exceeds this."
+        },
+        { .key  = {"mode"},
+          .value = {"server", "client"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "Set on the basis of where the xlator is loaded."
+        },
+        { .key = {"debug"},
+          .default_value = "false",
+          .type = GF_OPTION_TYPE_BOOL,
+          .description = "This is used in testing. Will dump compressed data \
+                          to disk as a gzip file."
+        },
+        { .key  = {NULL}
+        },
+};
diff --git a/xlators/features/compress/src/cdc.h b/xlators/features/compress/src/cdc.h
new file mode 100644
index 000000000..71f4d2317
--- /dev/null
+++ b/xlators/features/compress/src/cdc.h
@@ -0,0 +1,107 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CDC_H
+#define __CDC_H
+
+#ifdef HAVE_LIB_Z
+#include "zlib.h"
+#endif
+
+#include "xlator.h"
+
+#ifndef MAX_IOVEC
+#define MAX_IOVEC 16
+#endif
+
+typedef struct cdc_priv {
+        int window_size;
+        int mem_level;
+        int cdc_level;
+        int min_file_size;
+        int op_mode;
+        gf_boolean_t debug;
+        gf_lock_t lock;
+} cdc_priv_t;
+
+typedef struct cdc_info {
+        /* input bits */
+        int            count;
+        int32_t        ibytes;
+        struct iovec  *vector;
+        struct iatt   *buf;
+
+        /* output bits */
+        int            ncount;
+        int            nbytes;
+        int            buffer_size;
+        struct iovec   vec[MAX_IOVEC];
+        struct iobref *iobref;
+
+        /* zlib bits */
+#ifdef HAVE_LIB_Z
+        z_stream      stream;
+#endif
+        unsigned long crc;
+} cdc_info_t;
+
+#define NVEC(ci) (ci->ncount - 1)
+#define CURR_VEC(ci) ci->vec[ci->ncount - 1]
+#define THIS_VEC(ci, i) ci->vector[i]
+
+/* Gzip defaults */
+#define GF_CDC_DEF_WINDOWSIZE  -15 /* default value */
+#define GF_CDC_MAX_WINDOWSIZE  -8  /* max value     */
+
+#ifdef HAVE_LIB_Z
+#define GF_CDC_DEF_COMPRESSION Z_DEFAULT_COMPRESSION
+#else
+#define GF_CDC_DEF_COMPRESSION -1
+#endif
+
+#define GF_CDC_DEF_MEMLEVEL    8
+#define GF_CDC_DEF_BUFFERSIZE  262144 // 256K - default compression buffer size
+
+/* Operation mode
+ * If xlator is loaded on client, readv decompresses and writev compresses
+ * If xlator is loaded on server, readv compresses and writev decompresses
+ */
+#define GF_CDC_MODE_CLIENT   0
+#define GF_CDC_MODE_SERVER   1
+
+/* min size of data to do cmpression
+ * 0 == compress even 1byte
+ */
+#define GF_CDC_MIN_CHUNK_SIZE 0
+
+#define GF_CDC_VALIDATION_SIZE 8
+
+#define GF_CDC_OS_ID 0xFF
+#define GF_CDC_DEFLATE_CANARY_VAL "deflate"
+#define GF_CDC_DEBUG_DUMP_FILE "/tmp/cdcdump.gz"
+
+#define GF_CDC_MODE_IS_CLIENT(m) \
+        (strcmp (m, "client") == 0)
+
+#define GF_CDC_MODE_IS_SERVER(m) \
+        (strcmp (m, "server") == 0)
+
+int32_t
+cdc_compress (xlator_t *this,
+              cdc_priv_t *priv,
+              cdc_info_t *ci,
+              dict_t **xdata);
+int32_t
+cdc_decompress (xlator_t *this,
+                cdc_priv_t *priv,
+                cdc_info_t *ci,
+                dict_t *xdata);
+
+#endif
diff --git a/xlators/features/gfid-access/src/gfid-access.c b/xlators/features/gfid-access/src/gfid-access.c
index 755add522..da0ba7e50 100644
--- a/xlators/features/gfid-access/src/gfid-access.c
+++ b/xlators/features/gfid-access/src/gfid-access.c
@@ -68,8 +68,12 @@ ga_newfile_parse_args (xlator_t *this, data_t *data)
 
         min_len = sizeof (args->uid) + sizeof (args->gid) + sizeof (args->gfid)
                 + sizeof (args->st_mode) + 2 + 2;
-        if (blob_len < min_len)
+        if (blob_len < min_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid length: Total length is less "
+                        "than minimum length.");
                 goto err;
+        }
 
         args = mem_get0 (priv->newfile_args_pool);
         if (args == NULL)
@@ -93,7 +97,12 @@ ga_newfile_parse_args (xlator_t *this, data_t *data)
 
         len = strnlen (blob, blob_len);
         if (len == blob_len)
+        if (len == blob_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "gfid: %s. No null byte present.",
+                        args->gfid);
                 goto err;
+        }
 
         args->bname = GF_CALLOC (1, (len + 1), gf_common_mt_char);
         if (args->bname == NULL)
@@ -104,25 +113,39 @@ ga_newfile_parse_args (xlator_t *this, data_t *data)
         blob_len -= (len + 1);
 
         if (S_ISDIR (args->st_mode)) {
-                if (blob_len < sizeof (uint32_t))
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
                         goto err;
+                }
                 args->args.mkdir.mode = ntoh32 (*(uint32_t *)blob);
                 blob += sizeof (uint32_t);
                 blob_len -= sizeof (uint32_t);
 
-                if (blob_len < sizeof (uint32_t))
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
                         goto err;
+                }
                 args->args.mkdir.umask = ntoh32 (*(uint32_t *)blob);
                 blob += sizeof (uint32_t);
                 blob_len -= sizeof (uint32_t);
-                if (blob_len < 0)
+                if (blob_len < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
                         goto err;
-
+                }
         } else if (S_ISLNK (args->st_mode)) {
                 len = strnlen (blob, blob_len);
-                if (len == blob_len)
+                if (len == blob_len) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
                         goto err;
-
+                }
                 args->args.symlink.linkpath = GF_CALLOC (1, len + 1,
                                                          gf_common_mt_char);
                 if (args->args.symlink.linkpath == NULL)
@@ -132,27 +155,43 @@ ga_newfile_parse_args (xlator_t *this, data_t *data)
                 blob += (len + 1);
                 blob_len -= (len + 1);
         } else {
-                if (blob_len < sizeof (uint32_t))
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
                         goto err;
+                }
                 args->args.mknod.mode = ntoh32 (*(uint32_t *)blob);
                 blob += sizeof (uint32_t);
                 blob_len -= sizeof (uint32_t);
 
-                if (blob_len < sizeof (uint32_t))
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
                         goto err;
+                }
                 args->args.mknod.rdev = ntoh32 (*(uint32_t *)blob);
                 blob += sizeof (uint32_t);
                 blob_len -= sizeof (uint32_t);
 
-                if (blob_len < sizeof (uint32_t))
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
                         goto err;
+                }
                 args->args.mknod.umask = ntoh32 (*(uint32_t *)blob);
                 blob += sizeof (uint32_t);
                 blob_len -= sizeof (uint32_t);
         }
 
-        if (blob_len)
+        if (blob_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "gfid: %s. Invalid length",
+                        args->gfid);
                 goto err;
+        }
 
         return args;
 
@@ -469,13 +508,8 @@ ga_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
                 return 0;
         }
 
-        /* now, check if the setxattr() is on gfid-path */
-        if (!((loc->parent &&
-               __is_gfid_access_dir (loc->parent->gfid)) ||
-              __is_gfid_access_dir (loc->pargfid))) {
-                goto wind;
-        }
-
+        //If the inode is a virtual inode change the inode otherwise perform
+        //the operation on same inode
         GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
 
 wind:
diff --git a/xlators/features/index/src/index.c b/xlators/features/index/src/index.c
index 3245076ec..9253120f3 100644
--- a/xlators/features/index/src/index.c
+++ b/xlators/features/index/src/index.c
@@ -15,8 +15,10 @@
 #include "index.h"
 #include "options.h"
 #include "glusterfs3-xdr.h"
+#include "syncop.h"
 
 #define XATTROP_SUBDIR "xattrop"
+#define BASE_INDICES_HOLDER_SUBDIR "base_indices_holder"
 
 call_stub_t *
 __index_dequeue (struct list_head *callstubs)
@@ -242,20 +244,40 @@ check_delete_stale_index_file (xlator_t *this, char *filename)
 {
         int             ret = 0;
         struct stat     st = {0};
+        struct stat     base_index_st = {0};
         char            filepath[PATH_MAX] = {0};
+        char            filepath_under_base_indices_holder[PATH_MAX] = {0};
         index_priv_t    *priv = NULL;
 
         priv = this->private;
+        if (priv->to_be_healed_states != synced_state)
+                return;
+
         make_file_path (priv->index_basepath, XATTROP_SUBDIR,
                         filename, filepath, sizeof (filepath));
+
+        make_file_path (priv->index_basepath, BASE_INDICES_HOLDER_SUBDIR,
+                        filename, filepath_under_base_indices_holder,
+                        sizeof (filepath_under_base_indices_holder));
+
+
+        ret = stat (filepath_under_base_indices_holder, &base_index_st);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Base index is not created"
+                        "under index/base_indices_holder");
+                return;
+        }
+
         ret = stat (filepath, &st);
-        if (!ret && st.st_nlink == 1)
+        if (!ret && st.st_nlink == 2) {
                 unlink (filepath);
+                unlink (filepath_under_base_indices_holder);
+        }
 }
 
 static int
 index_fill_readdir (fd_t *fd, DIR *dir, off_t off,
-                    size_t size, gf_dirent_t *entries)
+                    size_t size, gf_dirent_t *entries, readdir_directory type)
 {
         off_t     in_case = -1;
         size_t    filled = 0;
@@ -298,7 +320,8 @@ index_fill_readdir (fd_t *fd, DIR *dir, off_t off,
                 }
 
                 if (!strncmp (entry->d_name, XATTROP_SUBDIR"-",
-                              strlen (XATTROP_SUBDIR"-"))) {
+                              strlen (XATTROP_SUBDIR"-")) &&
+                              (type == INDEX_XATTROP)) {
                         check_delete_stale_index_file (this, entry->d_name);
                         continue;
                 }
@@ -337,16 +360,175 @@ out:
 }
 
 int
+sync_base_indices (void *index_priv)
+{
+        index_priv_t    *priv = NULL;
+        DIR             *dir_base_holder  = NULL;
+        DIR             *xattrop_dir = NULL;
+        struct dirent   *entry = NULL;
+        char            base_indices_holder[PATH_MAX] = {0};
+        char            xattrop_directory[PATH_MAX] = {0};
+        char            base_index_path[PATH_MAX] = {0};
+        char            xattrop_index_path[PATH_MAX] = {0};
+        int             ret = 0;
+
+        priv = index_priv;
+
+        snprintf (base_indices_holder, PATH_MAX, "%s/%s", priv->index_basepath,
+                  BASE_INDICES_HOLDER_SUBDIR);
+        snprintf (xattrop_directory, PATH_MAX, "%s/%s", priv->index_basepath,
+                  XATTROP_SUBDIR);
+
+        if ((dir_base_holder = opendir(base_indices_holder)) == NULL) {
+                ret = -1;
+                goto out;
+        }
+        if ((xattrop_dir = opendir (xattrop_directory)) == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        priv->to_be_healed_states = sync_started;
+        while ((entry = readdir(xattrop_dir)) != NULL) {
+                if (!strcmp (entry->d_name, ".") ||
+                    !strcmp (entry->d_name, "..")) {
+                        continue;
+                }
+                if (strncmp (entry->d_name, XATTROP_SUBDIR"-",
+                    strlen (XATTROP_SUBDIR"-"))) {
+                        continue;
+                }
+                if (!strncmp (entry->d_name, XATTROP_SUBDIR"-",
+                    strlen (XATTROP_SUBDIR"-"))) {
+
+                    snprintf (xattrop_index_path, PATH_MAX, "%s/%s",
+                              xattrop_directory, entry->d_name);
+
+                    snprintf (base_index_path, PATH_MAX, "%s/%s",
+                              base_indices_holder, entry->d_name);
+
+                    ret = link (xattrop_index_path, base_index_path);
+                    if (ret && errno != EEXIST)
+                        goto out;
+
+                }
+        }
+        ret = closedir (xattrop_dir);
+        if (ret)
+                goto out;
+        ret = closedir (dir_base_holder);
+        if (ret)
+                goto out;
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
+int
+base_indices_syncing_done (int ret, call_frame_t *frame, void *data)
+{
+        index_priv_t         *priv = NULL;
+        priv = data;
+
+        if (!priv)
+                goto out;
+
+        if (ret) {
+                priv->to_be_healed_states = sync_not_started;
+        } else {
+                priv->to_be_healed_states = synced_state;
+        }
+
+        STACK_DESTROY (frame->root);
+
+out:
+        return 0;
+}
+
+int
+sync_base_indices_from_xattrop (xlator_t *this)
+{
+
+        index_priv_t          *priv = NULL;
+        char                  base_indices_holder[PATH_MAX] =  {0};
+        int                   ret = 0;
+        struct stat           st = {0};
+        DIR                   *dir = NULL;
+        struct dirent         *entry = NULL;
+        call_frame_t          *frame = NULL;
+
+        priv = this->private;
+
+        if (priv->to_be_healed_states != sync_not_started) {
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (base_indices_holder, PATH_MAX, "%s/%s", priv->index_basepath,
+                  BASE_INDICES_HOLDER_SUBDIR);
+
+        ret = stat (base_indices_holder, &st);
+
+        if (ret && (errno != ENOENT)) {
+                goto out;
+        } else if (errno == ENOENT) {
+                ret = index_dir_create (this, BASE_INDICES_HOLDER_SUBDIR);
+                if (ret)
+                        goto out;
+        } else {
+                if ((dir = opendir (base_indices_holder)) == NULL) {
+                        ret = -1;
+                        goto out;
+                }
+                while ((entry = readdir (dir)) != NULL) {
+                        if (!strcmp (entry->d_name, ".") ||
+                            !strcmp (entry->d_name,"..")) {
+                                continue;
+                        }
+                        ret = unlink (entry->d_name);
+                        if (ret)
+                                goto out;
+                }
+                closedir (dir);
+        }
+
+        /*At this point of time we have index/base_indicies_holder directory
+         *is with no entries*/
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame) {
+                ret = -1;
+                goto out;
+        }
+        set_lk_owner_from_ptr (&frame->root->lk_owner, frame->root);
+
+        frame->root->pid = LOW_PRIO_PROC_PID;
+
+        ret = synctask_new (this->ctx->env, sync_base_indices,
+                            base_indices_syncing_done,frame, priv);
+
+
+
+out:
+        return ret;
+
+}
+
+int
 index_add (xlator_t *this, uuid_t gfid, const char *subdir)
 {
         int32_t           op_errno = 0;
         char              gfid_path[PATH_MAX] = {0};
         char              index_path[PATH_MAX] = {0};
+        char              base_path[PATH_MAX] = {0};
         int               ret = 0;
         uuid_t            index = {0};
         index_priv_t      *priv = NULL;
         struct stat       st = {0};
         int               fd = 0;
+        int               index_created = 0;
 
         priv = this->private;
         GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !uuid_is_null (gfid),
@@ -364,9 +546,11 @@ index_add (xlator_t *this, uuid_t gfid, const char *subdir)
         ret = link (index_path, gfid_path);
         if (!ret || (errno == EEXIST))  {
                 ret = 0;
+                index_created = 1;
                 goto out;
         }
 
+
         op_errno = errno;
         if (op_errno == ENOENT) {
                 ret = index_dir_create (this, subdir);
@@ -398,10 +582,36 @@ index_add (xlator_t *this, uuid_t gfid, const char *subdir)
                         "add to index (%s)", uuid_utoa (gfid),
                         strerror (errno));
                 goto out;
+        } else {
+                index_created = 1;
+        }
+
+        if (priv->to_be_healed_states != sync_not_started) {
+                 make_index_path (priv->index_basepath,
+                                  GF_BASE_INDICES_HOLDER_GFID,
+                                  index, base_path, sizeof (base_path));
+                 ret = link (index_path, base_path);
+                 if (ret)
+                         goto out;
         }
 
         ret = 0;
 out:
+        /*If base_indices_holder is not created: create and sync
+         *If directory is present: delete contents and start syncing
+         *If syncing is in progress :No need to do any thing
+         *If syncing is done: No need to do anything*/
+        if (!ret) {
+                switch (priv->to_be_healed_states) {
+                        case sync_not_started:
+                                ret = sync_base_indices_from_xattrop (this);
+                                break;
+                        case sync_started:
+                        case synced_state:
+                                /*No need to do anything*/
+                                break;
+                }
+        }
         return ret;
 }
 
@@ -737,8 +947,18 @@ index_getxattr_wrapper (call_frame_t *frame, xlator_t *this,
                 goto done;
         }
 
-        ret = dict_set_static_bin (xattr, (char*)name, priv->xattrop_vgfid,
-                                   sizeof (priv->xattrop_vgfid));
+        if (!strcmp (name, GF_XATTROP_INDEX_GFID)) {
+
+                ret = dict_set_static_bin (xattr, (char*)name,
+                                           priv->xattrop_vgfid,
+                                           sizeof (priv->xattrop_vgfid));
+
+        } else if (!strcmp (name, GF_BASE_INDICES_HOLDER_GFID)) {
+
+                ret = dict_set_static_bin (xattr, (char*)name,
+                                           priv->base_indices_holder_vgfid,
+                                      sizeof (priv->base_indices_holder_vgfid));
+        }
         if (ret) {
                 ret = -ENOMEM;
                 gf_log (THIS->name, GF_LOG_ERROR, "xattrop index "
@@ -782,6 +1002,15 @@ index_lookup_wrapper (call_frame_t *frame, xlator_t *this,
         } else if (!uuid_compare (loc->pargfid, priv->xattrop_vgfid)) {
                 make_file_path (priv->index_basepath, XATTROP_SUBDIR,
                                 loc->name, path, sizeof (path));
+        } else if (!uuid_compare (loc->gfid,priv->base_indices_holder_vgfid)){
+                make_index_dir_path (priv->index_basepath,
+                                     BASE_INDICES_HOLDER_SUBDIR, path,
+                                    sizeof (path));
+                is_dir = _gf_true;
+        } else if (!uuid_compare (loc->pargfid, priv->base_indices_holder_vgfid)) {
+                make_file_path (priv->index_basepath,
+                                BASE_INDICES_HOLDER_SUBDIR,loc->name, path,
+                                sizeof (path));
         }
 
         ret = lstat (path, &lstatbuf);
@@ -803,10 +1032,14 @@ index_lookup_wrapper (call_frame_t *frame, xlator_t *this,
         }
 
         iatt_from_stat (&stbuf, &lstatbuf);
-        if (is_dir)
+        if (is_dir && !uuid_compare (loc->gfid, priv->xattrop_vgfid)) {
                 uuid_copy (stbuf.ia_gfid, priv->xattrop_vgfid);
-        else
+        } else if (is_dir &&
+                !uuid_compare (loc->gfid, priv->base_indices_holder_vgfid)) {
+                uuid_copy (stbuf.ia_gfid, priv->base_indices_holder_vgfid);
+        } else {
                 uuid_generate (stbuf.ia_gfid);
+        }
         stbuf.ia_ino = -1;
         op_ret = 0;
 done:
@@ -818,6 +1051,44 @@ done:
 }
 
 int32_t
+base_indices_readdir_wrapper (call_frame_t *frame, xlator_t *this,
+                              fd_t *fd, size_t size, off_t off, dict_t *xdata)
+{
+        index_priv_t    *priv = NULL;
+        char            base_indices_holder[PATH_MAX] = {0};
+        DIR             *dir = NULL;
+        int32_t         op_ret = -1;
+        int32_t         op_errno = 0;
+        int             count = 0;
+        gf_dirent_t     entries;
+
+        priv = this->private;
+
+        make_index_dir_path (priv->index_basepath, BASE_INDICES_HOLDER_SUBDIR,
+                             base_indices_holder, sizeof (base_indices_holder));
+
+        dir = opendir (base_indices_holder);
+        if (!dir) {
+                op_errno = EINVAL;
+                goto done;
+        }
+
+
+        INIT_LIST_HEAD (&entries.list);
+
+        count = index_fill_readdir (fd, dir, off, size, &entries,
+                                    BASE_INDICES_HOLDER);
+        /* pick ENOENT to indicate EOF */
+        op_errno = errno;
+        op_ret = count;
+        closedir (dir);
+done:
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, &entries, xdata);
+        gf_dirent_free (&entries);
+        return 0;
+}
+
+int32_t
 index_readdir_wrapper (call_frame_t *frame, xlator_t *this,
                        fd_t *fd, size_t size, off_t off, dict_t *xdata)
 {
@@ -848,7 +1119,8 @@ index_readdir_wrapper (call_frame_t *frame, xlator_t *this,
                 goto done;
         }
 
-        count = index_fill_readdir (fd, dir, off, size, &entries);
+        count = index_fill_readdir (fd, dir, off, size, &entries,
+                                    INDEX_XATTROP);
 
         /* pick ENOENT to indicate EOF */
         op_errno = errno;
@@ -915,7 +1187,10 @@ index_getxattr (call_frame_t *frame, xlator_t *this,
 {
         call_stub_t     *stub = NULL;
 
-        if (!name || strcmp (GF_XATTROP_INDEX_GFID, name))
+        if (!name)
+                goto out;
+        if (strcmp (GF_XATTROP_INDEX_GFID, name) &&
+            strcmp (GF_BASE_INDICES_HOLDER_GFID, name))
                 goto out;
 
         stub = fop_getxattr_stub (frame, index_getxattr_wrapper, loc, name,
@@ -942,7 +1217,9 @@ index_lookup (call_frame_t *frame, xlator_t *this,
         priv = this->private;
 
         if (uuid_compare (loc->gfid, priv->xattrop_vgfid) &&
-            uuid_compare (loc->pargfid, priv->xattrop_vgfid))
+            uuid_compare (loc->pargfid, priv->xattrop_vgfid) &&
+            uuid_compare (loc->gfid, priv->base_indices_holder_vgfid) &&
+            uuid_compare (loc->pargfid, priv->base_indices_holder_vgfid))
                 goto normal;
 
         stub = fop_lookup_stub (frame, index_lookup_wrapper, loc, xattr_req);
@@ -968,10 +1245,19 @@ index_readdir (call_frame_t *frame, xlator_t *this,
         index_priv_t    *priv = NULL;
 
         priv = this->private;
-        if (uuid_compare (fd->inode->gfid, priv->xattrop_vgfid))
+        if (uuid_compare (fd->inode->gfid, priv->xattrop_vgfid) &&
+            uuid_compare (fd->inode->gfid, priv->base_indices_holder_vgfid))
                 goto out;
-        stub = fop_readdir_stub (frame, index_readdir_wrapper, fd, size, off,
-                                 xdata);
+
+        if (!uuid_compare (fd->inode->gfid, priv->xattrop_vgfid)) {
+                stub = fop_readdir_stub (frame, index_readdir_wrapper, fd, size, 
+                                         off, xdata);
+        } else if (!uuid_compare (fd->inode->gfid,
+                                  priv->base_indices_holder_vgfid)) {
+                stub = fop_readdir_stub (frame, base_indices_readdir_wrapper,
+                                         fd, size, off, xdata);
+        }
+
         if (!stub) {
                 STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL, NULL);
                 return 0;
@@ -1075,10 +1361,13 @@ init (xlator_t *this)
         GF_OPTION_INIT ("index-base", priv->index_basepath, path, out);
         uuid_generate (priv->index);
         uuid_generate (priv->xattrop_vgfid);
+        /*base_indices_holder is a directory which contains hard links to
+         * all base indices inside indices/xattrop directory*/
+        uuid_generate (priv->base_indices_holder_vgfid);
         INIT_LIST_HEAD (&priv->callstubs);
 
         this->private = priv;
-        ret = pthread_create (&thread, &w_attr, index_worker, this);
+        ret = gf_thread_create (&thread, &w_attr, index_worker, this);
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING, "Failed to create "
                         "worker thread, aborting");
diff --git a/xlators/features/index/src/index.h b/xlators/features/index/src/index.h
index 661dcdbc4..d6dcb1c23 100644
--- a/xlators/features/index/src/index.h
+++ b/xlators/features/index/src/index.h
@@ -36,14 +36,28 @@ typedef struct index_fd_ctx {
         DIR *dir;
 } index_fd_ctx_t;
 
+typedef enum {
+        sync_not_started,
+        sync_started,
+        synced_state,
+} to_be_healed_states_t;
+
+typedef enum {
+        INDEX_XATTROP,
+        BASE_INDICES_HOLDER,
+} readdir_directory;
+
 typedef struct index_priv {
         char *index_basepath;
         uuid_t index;
         gf_lock_t lock;
         uuid_t xattrop_vgfid;//virtual gfid of the xattrop index dir
+        uuid_t base_indices_holder_vgfid; //virtual gfid of the
+                                          //to_be_healed_xattrop directory
         struct list_head callstubs;
         pthread_mutex_t mutex;
         pthread_cond_t  cond;
+        to_be_healed_states_t to_be_healed_states;
 } index_priv_t;
 
 #define INDEX_STACK_UNWIND(fop, frame, params ...)      \
diff --git a/xlators/features/locks/src/Makefile.am b/xlators/features/locks/src/Makefile.am
index 8908c1f52..0f79731b4 100644
--- a/xlators/features/locks/src/Makefile.am
+++ b/xlators/features/locks/src/Makefile.am
@@ -11,6 +11,7 @@ noinst_HEADERS = locks.h common.h locks-mem-types.h clear.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
 
+
 AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/features/locks/src/common.c b/xlators/features/locks/src/common.c
index bc4ea53f9..b3309580d 100644
--- a/xlators/features/locks/src/common.c
+++ b/xlators/features/locks/src/common.c
@@ -35,6 +35,7 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock);
 static int
 pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
                         posix_lock_t *old_lock);
+
 static pl_dom_list_t *
 __allocate_domain (const char *volume)
 {
@@ -75,8 +76,8 @@ get_domain (pl_inode_t *pl_inode, const char *volume)
 {
         pl_dom_list_t *dom = NULL;
 
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, pl_inode, out);
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, volume, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", pl_inode, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", volume, out);
 
         pthread_mutex_lock (&pl_inode->mutex);
         {
@@ -92,9 +93,9 @@ get_domain (pl_inode_t *pl_inode, const char *volume)
 unlock:
         pthread_mutex_unlock (&pl_inode->mutex);
         if (dom) {
-                gf_log (POSIX_LOCKS, GF_LOG_TRACE, "Domain %s found", volume);
+                gf_log ("posix-locks", GF_LOG_TRACE, "Domain %s found", volume);
         } else {
-                gf_log (POSIX_LOCKS, GF_LOG_TRACE, "Domain %s not found", volume);
+                gf_log ("posix-locks", GF_LOG_TRACE, "Domain %s not found", volume);
         }
 out:
         return dom;
@@ -138,7 +139,7 @@ pl_print_locker (char *str, int size, xlator_t *this, call_frame_t *frame)
         snprintf (str, size, "Pid=%llu, lk-owner=%s, Client=%p, Frame=%llu",
                   (unsigned long long) frame->root->pid,
                   lkowner_utoa (&frame->root->lk_owner),
-                  frame->root->trans,
+                  frame->root->client,
                   (unsigned long long) frame->root->unique);
 }
 
@@ -462,14 +463,14 @@ unlock:
 
 /* Create a new posix_lock_t */
 posix_lock_t *
-new_posix_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
+new_posix_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
                 gf_lkowner_t *owner, fd_t *fd)
 {
         posix_lock_t *lock = NULL;
 
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, flock, out);
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, transport, out);
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, fd, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", flock, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", client, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", fd, out);
 
         lock = GF_CALLOC (1, sizeof (posix_lock_t),
                           gf_locks_mt_posix_lock_t);
@@ -485,7 +486,7 @@ new_posix_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
         else
                 lock->fl_end = flock->l_start + flock->l_len - 1;
 
-        lock->transport  = transport;
+        lock->client     = client;
         lock->fd_num     = fd_to_fdnum (fd);
         lock->fd         = fd;
         lock->client_pid = client_pid;
@@ -565,7 +566,7 @@ same_owner (posix_lock_t *l1, posix_lock_t *l2)
 {
 
         return (is_same_lkowner (&l1->owner, &l2->owner) &&
-                (l1->transport  == l2->transport));
+                (l1->client == l2->client));
 
 }
 
@@ -694,7 +695,7 @@ subtract_locks (posix_lock_t *big, posix_lock_t *small)
         }
 
         GF_ASSERT (0);
-        gf_log (POSIX_LOCKS, GF_LOG_ERROR, "Unexpected case in subtract_locks");
+        gf_log ("posix-locks", GF_LOG_ERROR, "Unexpected case in subtract_locks");
 
 out:
         if (v.locks[0]) {
@@ -812,7 +813,7 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
                                 sum = add_locks (lock, conf);
 
                                 sum->fl_type    = lock->fl_type;
-                                sum->transport  = lock->transport;
+                                sum->client     = lock->client;
                                 sum->fd_num     = lock->fd_num;
                                 sum->client_pid = lock->client_pid;
                                 sum->owner      = lock->owner;
@@ -830,7 +831,7 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
                                 sum = add_locks (lock, conf);
 
                                 sum->fl_type    = conf->fl_type;
-                                sum->transport  = conf->transport;
+                                sum->client     = conf->client;
                                 sum->fd_num     = conf->fd_num;
                                 sum->client_pid = conf->client_pid;
                                 sum->owner      = conf->owner;
@@ -988,7 +989,7 @@ pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
         flock.l_len    = old_lock->user_flock.l_len;
 
 
-        unlock_lock = new_posix_lock (&flock, old_lock->transport,
+        unlock_lock = new_posix_lock (&flock, old_lock->client,
                                       old_lock->client_pid, &old_lock->owner,
                                       old_lock->fd);
         GF_VALIDATE_OR_GOTO (this->name, unlock_lock, out);
@@ -1097,3 +1098,124 @@ pl_getlk (pl_inode_t *pl_inode, posix_lock_t *lock)
 
         return conf;
 }
+
+
+struct _lock_table *
+pl_lock_table_new (void)
+{
+        struct _lock_table *new = NULL;
+
+        new = GF_CALLOC (1, sizeof (struct _lock_table), gf_common_mt_lock_table);
+        if (new == NULL) {
+                goto out;
+        }
+        INIT_LIST_HEAD (&new->entrylk_lockers);
+        INIT_LIST_HEAD (&new->inodelk_lockers);
+        LOCK_INIT (&new->lock);
+out:
+        return new;
+}
+
+
+int
+pl_add_locker (struct _lock_table *table, const char *volume,
+               loc_t *loc, fd_t *fd, pid_t pid, gf_lkowner_t *owner,
+               glusterfs_fop_t type)
+{
+        int32_t         ret = -1;
+        struct _locker *new = NULL;
+
+        GF_VALIDATE_OR_GOTO ("lock-table", table, out);
+        GF_VALIDATE_OR_GOTO ("lock-table", volume, out);
+
+        new = GF_CALLOC (1, sizeof (struct _locker), gf_common_mt_locker);
+        if (new == NULL) {
+                goto out;
+        }
+        INIT_LIST_HEAD (&new->lockers);
+
+        new->volume = gf_strdup (volume);
+
+        if (fd == NULL) {
+                loc_copy (&new->loc, loc);
+        } else {
+                new->fd = fd_ref (fd);
+        }
+
+        new->pid   = pid;
+        new->owner = *owner;
+
+        LOCK (&table->lock);
+        {
+                if (type == GF_FOP_ENTRYLK)
+                        list_add_tail (&new->lockers, &table->entrylk_lockers);
+                else
+                        list_add_tail (&new->lockers, &table->inodelk_lockers);
+        }
+        UNLOCK (&table->lock);
+out:
+        return ret;
+}
+
+int
+pl_del_locker (struct _lock_table *table, const char *volume,
+               loc_t *loc, fd_t *fd, gf_lkowner_t *owner, glusterfs_fop_t type)
+{
+        struct _locker    *locker = NULL;
+        struct _locker    *tmp = NULL;
+        int32_t            ret = -1;
+        struct list_head  *head = NULL;
+        struct list_head   del;
+
+        GF_VALIDATE_OR_GOTO ("lock-table", table, out);
+        GF_VALIDATE_OR_GOTO ("lock-table", volume, out);
+
+        INIT_LIST_HEAD (&del);
+
+        LOCK (&table->lock);
+        {
+                if (type == GF_FOP_ENTRYLK) {
+                        head = &table->entrylk_lockers;
+                } else {
+                        head = &table->inodelk_lockers;
+                }
+
+                list_for_each_entry_safe (locker, tmp, head, lockers) {
+                        if (!is_same_lkowner (&locker->owner, owner) ||
+                            strcmp (locker->volume, volume))
+                                continue;
+
+                        /*
+                         * It is possible for inodelk lock to come on anon-fd
+                         * and inodelk unlock to come on normal fd in case of
+                         * client re-opens. So don't check for fds to be equal.
+                         */
+                        if (locker->fd && fd)
+                                list_move_tail (&locker->lockers, &del);
+                        else if (locker->loc.inode && loc &&
+                                 (locker->loc.inode == loc->inode))
+                                list_move_tail (&locker->lockers, &del);
+                }
+        }
+        UNLOCK (&table->lock);
+
+        tmp = NULL;
+        locker = NULL;
+
+        list_for_each_entry_safe (locker, tmp, &del, lockers) {
+                list_del_init (&locker->lockers);
+                if (locker->fd)
+                        fd_unref (locker->fd);
+                else
+                        loc_wipe (&locker->loc);
+
+                GF_FREE (locker->volume);
+                GF_FREE (locker);
+        }
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
diff --git a/xlators/features/locks/src/common.h b/xlators/features/locks/src/common.h
index 4fbac5935..db19ec978 100644
--- a/xlators/features/locks/src/common.h
+++ b/xlators/features/locks/src/common.h
@@ -14,7 +14,7 @@
 /*dump locks format strings */
 #define RANGE_FMT               "type=%s, whence=%hd, start=%llu, len=%llu"
 #define ENTRY_FMT               "type=%s on basename=%s"
-#define DUMP_GEN_FMT            "pid = %llu, owner=%s, transport=%p"
+#define DUMP_GEN_FMT            "pid = %llu, owner=%s, client=%p"
 #define GRNTD_AT                "granted at %s"
 #define BLKD_AT                 "blocked at %s"
 #define CONN_ID                 "connection-id=%s"
@@ -31,8 +31,24 @@
 #define RANGE_BLKD_GRNTD_FMT    RANGE_FMT", "DUMP_BLKD_GRNTD_FMT
 
 #define SET_FLOCK_PID(flock, lock) ((flock)->l_pid = lock->client_pid)
+
+struct _locker {
+        struct list_head  lockers;
+        char             *volume;
+        loc_t             loc;
+        fd_t             *fd;
+        gf_lkowner_t      owner;
+        pid_t             pid;
+};
+
+struct _lock_table {
+        struct list_head  inodelk_lockers;
+        struct list_head  entrylk_lockers;
+        gf_lock_t         lock;
+};
+
 posix_lock_t *
-new_posix_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
+new_posix_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
                 gf_lkowner_t *owner, fd_t *fd);
 
 pl_inode_t *
@@ -146,6 +162,26 @@ pl_verify_reservelk (xlator_t *this, pl_inode_t *pl_inode,
                      posix_lock_t *lock, int can_block);
 int
 pl_reserve_unlock (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *reqlock);
+
 uint32_t
 check_entrylk_on_basename (xlator_t *this, inode_t *parent, char *basename);
+
+int32_t
+pl_add_locker (struct _lock_table *table, const char *volume,
+               loc_t *loc,
+               fd_t *fd,
+               pid_t pid,
+               gf_lkowner_t *owner,
+               glusterfs_fop_t type);
+
+int32_t
+pl_del_locker (struct _lock_table *table, const char *volume,
+               loc_t *loc,
+               fd_t *fd,
+               gf_lkowner_t *owner,
+               glusterfs_fop_t type);
+
+struct _lock_table *
+pl_lock_table_new (void);
+
 #endif /* __COMMON_H__ */
diff --git a/xlators/features/locks/src/entrylk.c b/xlators/features/locks/src/entrylk.c
index 6b649437e..0785dc547 100644
--- a/xlators/features/locks/src/entrylk.c
+++ b/xlators/features/locks/src/entrylk.c
@@ -25,7 +25,7 @@
 
 static pl_entry_lock_t *
 new_entrylk_lock (pl_inode_t *pinode, const char *basename, entrylk_type type,
-                  void *trans, pid_t client_pid, gf_lkowner_t *owner,
+                  client_t *client, pid_t client_pid, gf_lkowner_t *owner,
                   const char *volume)
 
 {
@@ -39,7 +39,7 @@ new_entrylk_lock (pl_inode_t *pinode, const char *basename, entrylk_type type,
 
         newlock->basename   = basename ? gf_strdup (basename) : NULL;
         newlock->type       = type;
-        newlock->trans      = trans;
+        newlock->trans      = client;
         newlock->volume     = volume;
         newlock->client_pid = client_pid;
         newlock->owner      = *owner;
@@ -310,14 +310,10 @@ __lock_name (pl_inode_t *pinode, const char *basename, entrylk_type type,
 {
         pl_entry_lock_t *lock       = NULL;
         pl_entry_lock_t *conf       = NULL;
-        void            *trans      = NULL;
-        pid_t            client_pid = 0;
         int              ret        = -EINVAL;
 
-        trans = frame->root->trans;
-        client_pid = frame->root->pid;
-
-        lock = new_entrylk_lock (pinode, basename, type, trans, client_pid,
+        lock = new_entrylk_lock (pinode, basename, type,
+                                 frame->root->client, frame->root->pid,
                                  &frame->root->lk_owner, dom->domain);
         if (!lock) {
                 ret = -ENOMEM;
@@ -326,7 +322,7 @@ __lock_name (pl_inode_t *pinode, const char *basename, entrylk_type type,
 
         lock->frame   = frame;
         lock->this    = this;
-        lock->trans   = trans;
+        lock->trans   = frame->root->client;
 
         if (conn_id) {
                 lock->connection_id = gf_strdup (conn_id);
@@ -531,8 +527,8 @@ grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
                 STACK_UNWIND_STRICT (entrylk, lock->frame, 0, 0, NULL);
 
                 GF_FREE (lock->connection_id);
-		GF_FREE ((char *)lock->basename);
-		GF_FREE (lock);
+                GF_FREE ((char *)lock->basename);
+                GF_FREE (lock);
         }
 
         GF_FREE ((char *)unlocked->basename);
@@ -543,13 +539,13 @@ grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
 }
 
 /**
- * release_entry_locks_for_transport: release all entry locks from this
- * transport for this loc_t
+ * release_entry_locks_for_client: release all entry locks from this
+ * client for this loc_t
  */
 
 static int
-release_entry_locks_for_transport (xlator_t *this, pl_inode_t *pinode,
-                                   pl_dom_list_t *dom, void *trans)
+release_entry_locks_for_client (xlator_t *this, pl_inode_t *pinode,
+                                pl_dom_list_t *dom, client_t *client)
 {
         pl_entry_lock_t  *lock = NULL;
         pl_entry_lock_t  *tmp = NULL;
@@ -563,14 +559,14 @@ release_entry_locks_for_transport (xlator_t *this, pl_inode_t *pinode,
         {
                 list_for_each_entry_safe (lock, tmp, &dom->blocked_entrylks,
                                           blocked_locks) {
-                        if (lock->trans != trans)
+                        if (lock->trans != client)
                                 continue;
 
                         list_del_init (&lock->blocked_locks);
 
                         gf_log (this->name, GF_LOG_TRACE,
                                 "releasing lock on  held by "
-                                "{transport=%p}",trans);
+                                "{client=%p}", client);
 
                         list_add (&lock->blocked_locks, &released);
 
@@ -578,14 +574,14 @@ release_entry_locks_for_transport (xlator_t *this, pl_inode_t *pinode,
 
                 list_for_each_entry_safe (lock, tmp, &dom->entrylk_list,
                                           domain_list) {
-                        if (lock->trans != trans)
+                        if (lock->trans != client)
                                 continue;
 
                         list_del_init (&lock->domain_list);
 
                         gf_log (this->name, GF_LOG_TRACE,
                                 "releasing lock on  held by "
-                                "{transport=%p}",trans);
+                                "{client=%p}", client);
 
                         GF_FREE ((char *)lock->basename);
                         GF_FREE (lock->connection_id);
@@ -630,19 +626,16 @@ pl_common_entrylk (call_frame_t *frame, xlator_t *this,
                    dict_t *xdata)
 
 {
-        int32_t  op_ret   = -1;
-        int32_t  op_errno = 0;
-
-        void *        transport = NULL;
-
-        pl_inode_t *     pinode   = NULL;
+        int32_t          op_ret   = -1;
+        int32_t          op_errno = 0;
         int              ret      = -1;
-        pl_entry_lock_t *unlocked = NULL;
         char             unwind   = 1;
-
-        pl_dom_list_t          *dom = NULL;
-        char            *conn_id  = NULL;
         GF_UNUSED int    dict_ret = -1;
+        pl_inode_t      *pinode   = NULL;
+        pl_entry_lock_t *unlocked = NULL;
+        pl_dom_list_t   *dom      = NULL;
+        char            *conn_id  = NULL;
+        pl_ctx_t        *ctx      = NULL;
 
         if (xdata)
                 dict_ret = dict_get_str (xdata, "connection-id", &conn_id);
@@ -661,19 +654,17 @@ pl_common_entrylk (call_frame_t *frame, xlator_t *this,
 
         entrylk_trace_in (this, frame, volume, fd, loc, basename, cmd, type);
 
-        transport = frame->root->trans;
-
         if (frame->root->lk_owner.len == 0) {
                 /*
                   this is a special case that means release
-                  all locks from this transport
+                  all locks from this client
                 */
 
                 gf_log (this->name, GF_LOG_TRACE,
-                        "Releasing locks for transport %p", transport);
+                        "Releasing locks for client %p", frame->root->client);
 
-                release_entry_locks_for_transport (this, pinode, dom,
-                                                   transport);
+                release_entry_locks_for_client (this, pinode, dom,
+                                                frame->root->client);
                 op_ret = 0;
 
                 goto out;
@@ -746,6 +737,24 @@ out:
                 entrylk_trace_out (this, frame, volume, fd, loc, basename,
                                    cmd, type, op_ret, op_errno);
 
+                ctx = pl_ctx_get (frame->root->client, this);
+
+                if (ctx == NULL) {
+                        gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+                        goto unwind;
+                }
+
+                if (cmd == ENTRYLK_UNLOCK)
+                        pl_del_locker (ctx->ltable, volume, loc, fd,
+                                       &frame->root->lk_owner,
+                                       GF_FOP_ENTRYLK);
+                else
+                        pl_add_locker (ctx->ltable, volume, loc, fd,
+                                       frame->root->pid,
+                                       &frame->root->lk_owner,
+                                       GF_FOP_ENTRYLK);
+
+unwind:
                 STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, NULL);
         } else {
                 entrylk_trace_block (this, frame, volume, fd, loc, basename,
@@ -767,7 +776,6 @@ pl_entrylk (call_frame_t *frame, xlator_t *this,
             const char *volume, loc_t *loc, const char *basename,
             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-
         pl_common_entrylk (frame, this, volume, loc->inode, basename, cmd,
                            type, loc, NULL, xdata);
 
@@ -786,7 +794,6 @@ pl_fentrylk (call_frame_t *frame, xlator_t *this,
              const char *volume, fd_t *fd, const char *basename,
              entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-
         pl_common_entrylk (frame, this, volume, fd->inode, basename, cmd,
                            type, NULL, fd, xdata);
 
diff --git a/xlators/features/locks/src/inodelk.c b/xlators/features/locks/src/inodelk.c
index 7b5c17033..508523e11 100644
--- a/xlators/features/locks/src/inodelk.c
+++ b/xlators/features/locks/src/inodelk.c
@@ -124,7 +124,7 @@ static inline int
 same_inodelk_owner (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
         return (is_same_lkowner (&l1->owner, &l2->owner) &&
-                (l1->transport  == l2->transport));
+                (l1->client == l2->client));
 }
 
 /* Returns true if the 2 inodelks conflict with each other */
@@ -294,7 +294,7 @@ __inode_unlock_lock (xlator_t *this, pl_inode_lock_t *lock, pl_dom_list_t *dom)
                         " Matching lock not found for unlock %llu-%llu, by %s "
                         "on %p", (unsigned long long)lock->fl_start,
                         (unsigned long long)lock->fl_end,
-                        lkowner_utoa (&lock->owner), lock->transport);
+                        lkowner_utoa (&lock->owner), lock->client);
                 goto out;
         }
         __delete_inode_lock (conf);
@@ -302,7 +302,7 @@ __inode_unlock_lock (xlator_t *this, pl_inode_lock_t *lock, pl_dom_list_t *dom)
                 " Matching lock found for unlock %llu-%llu, by %s on %p",
                 (unsigned long long)lock->fl_start,
                 (unsigned long long)lock->fl_end, lkowner_utoa (&lock->owner),
-                lock->transport);
+                lock->client);
 
 out:
         return conf;
@@ -375,10 +375,10 @@ grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
         pthread_mutex_unlock (&pl_inode->mutex);
 }
 
-/* Release all inodelks from this transport */
+/* Release all inodelks from this client */
 static int
-release_inode_locks_of_transport (xlator_t *this, pl_dom_list_t *dom,
-                                  inode_t *inode, void *trans)
+release_inode_locks_of_client (xlator_t *this, pl_dom_list_t *dom,
+                                  inode_t *inode, client_t *client)
 {
         pl_inode_lock_t *tmp = NULL;
         pl_inode_lock_t *l = NULL;
@@ -398,7 +398,7 @@ release_inode_locks_of_transport (xlator_t *this, pl_dom_list_t *dom,
         {
 
                 list_for_each_entry_safe (l, tmp, &dom->blocked_inodelks, blocked_locks) {
-                        if (l->transport != trans)
+                        if (l->client != client)
                                 continue;
 
                         list_del_init (&l->blocked_locks);
@@ -411,8 +411,8 @@ release_inode_locks_of_transport (xlator_t *this, pl_dom_list_t *dom,
 
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "releasing blocking lock on %s held by "
-                                "{transport=%p, pid=%"PRId64" lk-owner=%s}",
-                                file, trans, (uint64_t) l->client_pid,
+                                "{client=%p, pid=%"PRId64" lk-owner=%s}",
+                                file, client, (uint64_t) l->client_pid,
                                 lkowner_utoa (&l->owner));
 
                         list_add (&l->blocked_locks, &released);
@@ -423,7 +423,7 @@ release_inode_locks_of_transport (xlator_t *this, pl_dom_list_t *dom,
                 }
 
                 list_for_each_entry_safe (l, tmp, &dom->inodelk_list, list) {
-                        if (l->transport != trans)
+                        if (l->client != client)
                                 continue;
 
                         inode_path (inode, NULL, &path);
@@ -434,8 +434,8 @@ release_inode_locks_of_transport (xlator_t *this, pl_dom_list_t *dom,
 
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "releasing granted lock on %s held by "
-                                "{transport=%p, pid=%"PRId64" lk-owner=%s}",
-                                file, trans, (uint64_t) l->client_pid,
+                                "{client=%p, pid=%"PRId64" lk-owner=%s}",
+                                file, client, (uint64_t) l->client_pid,
                                 lkowner_utoa (&l->owner));
 
                         if (path) {
@@ -518,7 +518,7 @@ out:
 
 /* Create a new inode_lock_t */
 pl_inode_lock_t *
-new_inode_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
+new_inode_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
                 call_frame_t *frame, xlator_t *this, const char *volume,
                 char *conn_id)
 
@@ -539,7 +539,7 @@ new_inode_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
         else
                 lock->fl_end = flock->l_start + flock->l_len - 1;
 
-        lock->transport  = transport;
+        lock->client     = client;
         lock->client_pid = client_pid;
         lock->volume     = volume;
         lock->owner      = frame->root->lk_owner;
@@ -599,14 +599,13 @@ pl_common_inodelk (call_frame_t *frame, xlator_t *this,
         int               ret        = -1;
         GF_UNUSED int     dict_ret   = -1;
         int               can_block  = 0;
-        pid_t             client_pid = -1;
-        void *            transport  = NULL;
         pl_inode_t *      pinode     = NULL;
         pl_inode_lock_t * reqlock    = NULL;
         pl_dom_list_t *   dom        = NULL;
         char             *res        = NULL;
         char             *res1       = NULL;
-        char             *conn_id     = NULL;
+        char             *conn_id    = NULL;
+        pl_ctx_t         *ctx        = NULL;
 
         if (xdata)
                 dict_ret = dict_get_str (xdata, "connection-id", &conn_id);
@@ -628,9 +627,6 @@ pl_common_inodelk (call_frame_t *frame, xlator_t *this,
 
         pl_trace_in (this, frame, fd, loc, cmd, flock, volume);
 
-        transport  = frame->root->trans;
-        client_pid = frame->root->pid;
-
         pinode = pl_inode_get (this, inode);
         if (!pinode) {
                 op_errno = ENOMEM;
@@ -646,25 +642,25 @@ pl_common_inodelk (call_frame_t *frame, xlator_t *this,
         if (frame->root->lk_owner.len == 0) {
                 /*
                   special case: this means release all locks
-                  from this transport
+                  from this client
                 */
                 gf_log (this->name, GF_LOG_TRACE,
-                        "Releasing all locks from transport %p", transport);
+                        "Releasing all locks from client %p", frame->root->client);
 
-                release_inode_locks_of_transport (this, dom, inode, transport);
+                release_inode_locks_of_client (this, dom, inode, frame->root->client);
                 _pl_convert_volume (volume, &res1);
                 if (res1) {
                         dom = get_domain (pinode, res1);
                         if (dom)
-                                release_inode_locks_of_transport (this, dom,
-                                                        inode, transport);
+                                release_inode_locks_of_client (this, dom,
+                                                        inode, frame->root->client);
                 }
 
                 op_ret = 0;
                 goto unwind;
         }
 
-        reqlock = new_inode_lock (flock, transport, client_pid,
+        reqlock = new_inode_lock (flock, frame->root->client, frame->root->pid,
                                   frame, this, volume, conn_id);
 
         if (!reqlock) {
@@ -708,6 +704,23 @@ pl_common_inodelk (call_frame_t *frame, xlator_t *this,
 
         op_ret = 0;
 
+        ctx = pl_ctx_get (frame->root->client, this);
+
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+                goto unwind;
+        }
+
+        if (flock->l_type == F_UNLCK)
+                pl_del_locker (ctx->ltable, volume, loc, fd,
+                               &frame->root->lk_owner,
+                               GF_FOP_INODELK);
+        else
+                pl_add_locker (ctx->ltable, volume, loc, fd,
+                               frame->root->pid,
+                               &frame->root->lk_owner,
+                               GF_FOP_INODELK);
+
 unwind:
         if ((inode != NULL) && (flock !=NULL)) {
                 pl_update_refkeeper (this, inode);
@@ -726,9 +739,8 @@ pl_inodelk (call_frame_t *frame, xlator_t *this,
             const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock,
             dict_t *xdata)
 {
-
-        pl_common_inodelk (frame, this, volume, loc->inode, cmd, flock, loc, NULL,
-                           xdata);
+        pl_common_inodelk (frame, this, volume, loc->inode, cmd, flock,
+                           loc, NULL, xdata);
 
         return 0;
 }
@@ -738,9 +750,8 @@ pl_finodelk (call_frame_t *frame, xlator_t *this,
              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
              dict_t *xdata)
 {
-
-        pl_common_inodelk (frame, this, volume, fd->inode, cmd, flock, NULL, fd,
-                           xdata);
+        pl_common_inodelk (frame, this, volume, fd->inode, cmd, flock,
+                           NULL, fd, xdata);
 
         return 0;
 
diff --git a/xlators/features/locks/src/locks.h b/xlators/features/locks/src/locks.h
index 5bd0b05e0..76fc941d7 100644
--- a/xlators/features/locks/src/locks.h
+++ b/xlators/features/locks/src/locks.h
@@ -19,10 +19,10 @@
 #include "stack.h"
 #include "call-stub.h"
 #include "locks-mem-types.h"
+#include "client_t.h"
 
 #include "lkowner.h"
 
-#define POSIX_LOCKS "posix-locks"
 struct __pl_fd;
 
 struct __posix_lock {
@@ -33,7 +33,7 @@ struct __posix_lock {
         off_t              fl_end;
 
         short              blocked;    /* waiting to acquire */
-        struct gf_flock       user_flock; /* the flock supplied by the user */
+        struct gf_flock    user_flock; /* the flock supplied by the user */
         xlator_t          *this;       /* required for blocked locks */
         unsigned long      fd_num;
 
@@ -46,7 +46,7 @@ struct __posix_lock {
         /* These two together serve to uniquely identify each process
            across nodes */
 
-        void              *transport;     /* to identify client node */
+        void              *client;     /* to identify client node */
         gf_lkowner_t       owner;
         pid_t              client_pid;    /* pid of client process */
 };
@@ -63,7 +63,7 @@ struct __pl_inode_lock {
 
         const char        *volume;
 
-        struct gf_flock       user_flock; /* the flock supplied by the user */
+        struct gf_flock    user_flock; /* the flock supplied by the user */
         xlator_t          *this;       /* required for blocked locks */
         fd_t              *fd;
 
@@ -75,7 +75,7 @@ struct __pl_inode_lock {
         /* These two together serve to uniquely identify each process
            across nodes */
 
-        void              *transport;     /* to identify client node */
+        void              *client;     /* to identify client node */
         gf_lkowner_t       owner;
         pid_t              client_pid;    /* pid of client process */
 
@@ -156,6 +156,7 @@ typedef struct {
         char           *brickname;
 } posix_locks_private_t;
 
+
 typedef struct {
         gf_boolean_t   entrylk_count_req;
         gf_boolean_t   inodelk_count_req;
@@ -171,8 +172,21 @@ typedef struct {
         enum {TRUNCATE, FTRUNCATE} op;
 } pl_local_t;
 
+
 typedef struct {
         struct list_head locks_list;
 } pl_fdctx_t;
 
+
+typedef struct _locks_ctx {
+        gf_lock_t            ltable_lock; /* only for replace, 
+                                             ltable has its own internal 
+                                             lock for operations */
+        struct _lock_table  *ltable;
+} pl_ctx_t;
+
+
+pl_ctx_t *
+pl_ctx_get (client_t *client, xlator_t *xlator);
+
 #endif /* __POSIX_LOCKS_H__ */
diff --git a/xlators/features/locks/src/posix.c b/xlators/features/locks/src/posix.c
index 0d6f32fa3..7bfb38a51 100644
--- a/xlators/features/locks/src/posix.c
+++ b/xlators/features/locks/src/posix.c
@@ -51,7 +51,7 @@ pl_new_fdctx ()
 
         fdctx = GF_CALLOC (1, sizeof (*fdctx),
                            gf_locks_mt_pl_fdctx_t);
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, fdctx, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", fdctx, out);
 
         INIT_LIST_HEAD (&fdctx->locks_list);
 
@@ -66,7 +66,7 @@ pl_check_n_create_fdctx (xlator_t *this, fd_t *fd)
         uint64_t    tmp   = 0;
         pl_fdctx_t *fdctx = NULL;
 
-        GF_VALIDATE_OR_GOTO (POSIX_LOCKS, this, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", this, out);
         GF_VALIDATE_OR_GOTO (this->name, fd, out);
 
         LOCK (&fd->lock);
@@ -119,7 +119,7 @@ pl_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 static int
 truncate_allowed (pl_inode_t *pl_inode,
-                  void *transport, pid_t client_pid,
+                  client_t *client, pid_t client_pid,
                   gf_lkowner_t *owner, off_t offset)
 {
         posix_lock_t *l = NULL;
@@ -128,7 +128,7 @@ truncate_allowed (pl_inode_t *pl_inode,
 
         region.fl_start   = offset;
         region.fl_end     = LLONG_MAX;
-        region.transport  = transport;
+        region.client     = client;
         region.client_pid = client_pid;
         region.owner      = *owner;
 
@@ -139,7 +139,7 @@ truncate_allowed (pl_inode_t *pl_inode,
                             && locks_overlap (&region, l)
                             && !same_owner (&region, l)) {
                                 ret = 0;
-                                gf_log (POSIX_LOCKS, GF_LOG_TRACE, "Truncate "
+                                gf_log ("posix-locks", GF_LOG_TRACE, "Truncate "
                                         "allowed");
                                 break;
                         }
@@ -186,7 +186,7 @@ truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (priv->mandatory
             && pl_inode->mandatory
-            && !truncate_allowed (pl_inode, frame->root->trans,
+            && !truncate_allowed (pl_inode, frame->root->client,
                                   frame->root->pid, &frame->root->lk_owner,
                                   local->offset)) {
                 op_ret   = -1;
@@ -347,7 +347,7 @@ delete_locks_of_fd (xlator_t *this, pl_inode_t *pl_inode, fd_t *fd)
 
 static void
 __delete_locks_of_owner (pl_inode_t *pl_inode,
-                         void *transport, gf_lkowner_t *owner)
+                         client_t *client, gf_lkowner_t *owner)
 {
         posix_lock_t *tmp = NULL;
         posix_lock_t *l = NULL;
@@ -357,7 +357,7 @@ __delete_locks_of_owner (pl_inode_t *pl_inode,
         list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
                 if (l->blocked)
                         continue;
-                if ((l->transport == transport) &&
+                if ((l->client == client) &&
                     is_same_lkowner (&l->owner, owner)) {
                         gf_log ("posix-locks", GF_LOG_TRACE,
                                 " Flushing lock"
@@ -810,7 +810,7 @@ pl_migrate_locks (call_frame_t *frame, fd_t *newfd, uint64_t oldfd_num,
                 list_for_each_entry (l, &pl_inode->ext_list, list) {
                         if (l->fd_num == oldfd_num) {
                                 l->fd_num = newfd_num;
-                                l->transport = frame->root->trans;
+                                l->client = frame->root->client;
                         }
                 }
         }
@@ -983,7 +983,7 @@ pl_flush (call_frame_t *frame, xlator_t *this,
         }
         pthread_mutex_lock (&pl_inode->mutex);
         {
-                __delete_locks_of_owner (pl_inode, frame->root->trans,
+                __delete_locks_of_owner (pl_inode, frame->root->client,
                                          &frame->root->lk_owner);
         }
         pthread_mutex_unlock (&pl_inode->mutex);
@@ -1178,7 +1178,7 @@ pl_readv (call_frame_t *frame, xlator_t *this,
         if (priv->mandatory && pl_inode->mandatory) {
                 region.fl_start   = offset;
                 region.fl_end     = offset + size - 1;
-                region.transport  = frame->root->trans;
+                region.client     = frame->root->client;
                 region.fd_num     = fd_to_fdnum(fd);
                 region.client_pid = frame->root->pid;
                 region.owner      = frame->root->lk_owner;
@@ -1272,7 +1272,7 @@ pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         if (priv->mandatory && pl_inode->mandatory) {
                 region.fl_start   = offset;
                 region.fl_end     = offset + iov_length (vector, count) - 1;
-                region.transport  = frame->root->trans;
+                region.client     = frame->root->client;
                 region.fd_num     = fd_to_fdnum(fd);
                 region.client_pid = frame->root->pid;
                 region.owner      = frame->root->lk_owner;
@@ -1353,7 +1353,7 @@ lock_dup (posix_lock_t *lock)
 {
         posix_lock_t *new_lock = NULL;
 
-        new_lock = new_posix_lock (&lock->user_flock, lock->transport,
+        new_lock = new_posix_lock (&lock->user_flock, lock->client,
                                    lock->client_pid, &lock->owner,
                                    (fd_t *)lock->fd_num);
         return new_lock;
@@ -1513,8 +1513,6 @@ int
 pl_lk (call_frame_t *frame, xlator_t *this,
        fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        void         *transport  = NULL;
-        pid_t         client_pid = 0;
         pl_inode_t   *pl_inode   = NULL;
         int           op_ret     = 0;
         int           op_errno   = 0;
@@ -1523,9 +1521,6 @@ pl_lk (call_frame_t *frame, xlator_t *this,
         posix_lock_t *conf       = NULL;
         int           ret        = 0;
 
-        transport  = frame->root->trans;
-        client_pid = frame->root->pid;
-
         if ((flock->l_start < 0) || (flock->l_len < 0)) {
                 op_ret = -1;
                 op_errno = EINVAL;
@@ -1539,7 +1534,7 @@ pl_lk (call_frame_t *frame, xlator_t *this,
                 goto unwind;
         }
 
-        reqlock = new_posix_lock (flock, transport, client_pid,
+        reqlock = new_posix_lock (flock, frame->root->client, frame->root->pid,
                                   &frame->root->lk_owner, fd);
 
         if (!reqlock) {
@@ -2328,7 +2323,7 @@ __dump_inodelks (pl_inode_t *pl_inode)
                         SET_FLOCK_PID (&lock->user_flock, lock);
                         pl_dump_lock (tmp, 256, &lock->user_flock,
                                       &lock->owner,
-                                      lock->transport, lock->connection_id,
+                                      lock->client, lock->connection_id,
                                       &lock->granted_time.tv_sec,
                                       &lock->blkd_time.tv_sec,
                                       _gf_true);
@@ -2345,7 +2340,7 @@ __dump_inodelks (pl_inode_t *pl_inode)
                         SET_FLOCK_PID (&lock->user_flock, lock);
                         pl_dump_lock (tmp, 256, &lock->user_flock,
                                       &lock->owner,
-                                      lock->transport, lock->connection_id,
+                                      lock->client, lock->connection_id,
                                       0, &lock->blkd_time.tv_sec,
                                       _gf_false);
                         gf_proc_dump_write(key, tmp);
@@ -2386,7 +2381,7 @@ __dump_posixlks (pl_inode_t *pl_inode)
                                      count,
                                      lock->blocked ? "BLOCKED" : "ACTIVE");
               pl_dump_lock (tmp, 256, &lock->user_flock,
-                            &lock->owner, lock->transport, NULL,
+                            &lock->owner, lock->client, NULL,
                             &lock->granted_time.tv_sec, &lock->blkd_time.tv_sec,
                             (lock->blocked)? _gf_false: _gf_true);
               gf_proc_dump_write(key, tmp);
@@ -2510,6 +2505,124 @@ mem_acct_init (xlator_t *this)
         return ret;
 }
 
+
+pl_ctx_t*
+pl_ctx_get (client_t *client, xlator_t *xlator)
+{
+        void *tmp = NULL;
+        pl_ctx_t *ctx = NULL;
+
+        client_ctx_get (client, xlator, &tmp);
+
+        ctx = tmp;
+
+        if (ctx != NULL)
+                goto out;
+
+        ctx = GF_CALLOC (1, sizeof (pl_ctx_t), gf_locks_mt_posix_lock_t);
+
+        if (ctx == NULL)
+                goto out;
+
+        ctx->ltable = pl_lock_table_new();
+
+        if (ctx->ltable == NULL) {
+                GF_FREE (ctx);
+                ctx = NULL;
+                goto out;
+        }
+
+        LOCK_INIT (&ctx->ltable_lock);
+
+        if (client_ctx_set (client, xlator, ctx) != 0) {
+                LOCK_DESTROY (&ctx->ltable_lock);
+                GF_FREE (ctx->ltable);
+                GF_FREE (ctx);
+                ctx = NULL;
+        }
+out:
+        return ctx;
+}
+
+static void
+ltable_delete_locks (struct _lock_table *ltable)
+{
+        struct _locker *locker = NULL;
+        struct _locker *tmp    = NULL;
+
+        list_for_each_entry_safe (locker, tmp, &ltable->inodelk_lockers, lockers) {
+                if (locker->fd)
+                        pl_del_locker (ltable, locker->volume, &locker->loc,
+                                       locker->fd, &locker->owner,
+                                       GF_FOP_INODELK);
+                GF_FREE (locker->volume);
+                GF_FREE (locker);
+        }
+
+        list_for_each_entry_safe (locker, tmp, &ltable->entrylk_lockers, lockers) {
+                if (locker->fd)
+                        pl_del_locker (ltable, locker->volume, &locker->loc,
+                                       locker->fd, &locker->owner,
+                                       GF_FOP_ENTRYLK);
+                GF_FREE (locker->volume);
+                GF_FREE (locker);
+        }
+        GF_FREE (ltable);
+}
+
+
+static int32_t
+destroy_cbk (xlator_t *this, client_t *client)
+{
+        void     *tmp       = NULL;
+        pl_ctx_t *locks_ctx = NULL;
+
+        client_ctx_del (client, this, &tmp);
+
+        if (tmp == NULL)
+                return 0
+;
+        locks_ctx = tmp;
+        if (locks_ctx->ltable)
+                ltable_delete_locks (locks_ctx->ltable);
+
+        LOCK_DESTROY (&locks_ctx->ltable_lock);
+        GF_FREE (locks_ctx);
+
+        return 0;
+}
+
+
+static int32_t
+disconnect_cbk (xlator_t *this, client_t *client)
+{
+        int32_t              ret       = 0;
+        pl_ctx_t            *locks_ctx = NULL;
+        struct _lock_table  *ltable    = NULL;
+
+        locks_ctx = pl_ctx_get (client, this);
+        if (locks_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+                goto out;
+        }
+
+        LOCK (&locks_ctx->ltable_lock);
+        {
+                if (locks_ctx->ltable) {
+                        ltable = locks_ctx->ltable;
+                        locks_ctx->ltable = pl_lock_table_new ();
+                }
+        }
+        UNLOCK (&locks_ctx->ltable_lock);
+
+        if (ltable)
+                ltable_delete_locks (ltable);
+
+out:
+        return ret;
+}
+
+
 int
 init (xlator_t *this)
 {
@@ -2538,7 +2651,7 @@ init (xlator_t *this)
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "'locks' translator is not loaded over a storage "
                         "translator");
-                goto out;;
+                goto out;
         }
 
         priv = GF_CALLOC (1, sizeof (*priv),
@@ -2640,9 +2753,11 @@ struct xlator_dumpops dumpops = {
 };
 
 struct xlator_cbks cbks = {
-        .forget      = pl_forget,
-        .release     = pl_release,
-        .releasedir  = pl_releasedir,
+        .forget            = pl_forget,
+        .release           = pl_release,
+        .releasedir        = pl_releasedir,
+        .client_destroy    = destroy_cbk,
+        .client_disconnect = disconnect_cbk,
 };
 
 
diff --git a/xlators/features/marker/src/marker.c b/xlators/features/marker/src/marker.c
index 59152db43..6a2c85691 100644
--- a/xlators/features/marker/src/marker.c
+++ b/xlators/features/marker/src/marker.c
@@ -1953,6 +1953,73 @@ err:
         return 0;
 }
 
+int32_t
+marker_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred during zerofill",
+                        strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (zerofill, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_initiate_quota_txn (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               size_t len, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_zerofill_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (zerofill, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
+}
+
 
 /* when a call from the special client is received on
  * key trusted.glusterfs.volume-mark with value "RESET"
@@ -2778,6 +2845,7 @@ struct xlator_fops fops = {
         .readdirp    = marker_readdirp,
 	.fallocate   = marker_fallocate,
 	.discard     = marker_discard,
+        .zerofill    = marker_zerofill,
 };
 
 struct xlator_cbks cbks = {
diff --git a/xlators/features/qemu-block/src/Makefile.am b/xlators/features/qemu-block/src/Makefile.am
index 484c0f19a..08a7b62a0 100644
--- a/xlators/features/qemu-block/src/Makefile.am
+++ b/xlators/features/qemu-block/src/Makefile.am
@@ -9,6 +9,7 @@ qemu_block_la_SOURCES_qemu = \
 	$(CONTRIBDIR)/qemu/qemu-coroutine.c \
 	$(CONTRIBDIR)/qemu/qemu-coroutine-lock.c \
 	$(CONTRIBDIR)/qemu/qemu-coroutine-sleep.c \
+	$(CONTRIBDIR)/qemu/coroutine-ucontext.c \
 	$(CONTRIBDIR)/qemu/block.c \
 	$(CONTRIBDIR)/qemu/nop-symbols.c
 
@@ -74,6 +75,7 @@ noinst_HEADERS_qemu = \
 	$(CONTRIBDIR)/qemu/qmp-commands.h \
 	$(CONTRIBDIR)/qemu/trace/generated-tracers.h \
 	$(CONTRIBDIR)/qemu/include/config.h \
+	$(CONTRIBDIR)/qemu/include/glib-compat.h \
 	$(CONTRIBDIR)/qemu/include/qemu-common.h \
 	$(CONTRIBDIR)/qemu/include/trace.h \
 	$(CONTRIBDIR)/qemu/include/block/coroutine.h \
@@ -83,8 +85,13 @@ noinst_HEADERS_qemu = \
 	$(CONTRIBDIR)/qemu/include/block/blockjob.h \
 	$(CONTRIBDIR)/qemu/include/block/coroutine.h \
 	$(CONTRIBDIR)/qemu/include/block/coroutine_int.h \
+	$(CONTRIBDIR)/qemu/include/block/snapshot.h \
+	$(CONTRIBDIR)/qemu/include/exec/cpu-common.h \
+	$(CONTRIBDIR)/qemu/include/exec/hwaddr.h \
+	$(CONTRIBDIR)/qemu/include/exec/poison.h \
 	$(CONTRIBDIR)/qemu/include/fpu/softfloat.h \
 	$(CONTRIBDIR)/qemu/include/migration/migration.h \
+	$(CONTRIBDIR)/qemu/include/migration/qemu-file.h \
 	$(CONTRIBDIR)/qemu/include/migration/vmstate.h \
 	$(CONTRIBDIR)/qemu/include/monitor/monitor.h \
 	$(CONTRIBDIR)/qemu/include/monitor/readline.h \
@@ -97,6 +104,7 @@ noinst_HEADERS_qemu = \
 	$(CONTRIBDIR)/qemu/include/qapi/qmp/qerror.h \
 	$(CONTRIBDIR)/qemu/include/qapi/qmp/qfloat.h \
 	$(CONTRIBDIR)/qemu/include/qapi/qmp/qint.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qjson.h \
 	$(CONTRIBDIR)/qemu/include/qapi/qmp/qlist.h \
 	$(CONTRIBDIR)/qemu/include/qapi/qmp/qobject.h \
 	$(CONTRIBDIR)/qemu/include/qapi/qmp/qstring.h \
@@ -117,6 +125,7 @@ noinst_HEADERS_qemu = \
 	$(CONTRIBDIR)/qemu/include/qemu/notify.h \
 	$(CONTRIBDIR)/qemu/include/qemu/option.h \
 	$(CONTRIBDIR)/qemu/include/qemu/option_int.h \
+	$(CONTRIBDIR)/qemu/include/qemu/osdep.h \
 	$(CONTRIBDIR)/qemu/include/qemu/queue.h \
 	$(CONTRIBDIR)/qemu/include/qemu/sockets.h \
 	$(CONTRIBDIR)/qemu/include/qemu/thread-posix.h \
@@ -132,7 +141,6 @@ noinst_HEADERS = \
 	$(noinst_HEADERS_qemu) \
 	qemu-block.h \
 	qemu-block-memory-types.h \
-	coroutine-synctask.h \
 	qb-coroutines.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
diff --git a/xlators/features/qemu-block/src/bdrv-xlator.c b/xlators/features/qemu-block/src/bdrv-xlator.c
index 145f7c79d..106c59775 100644
--- a/xlators/features/qemu-block/src/bdrv-xlator.c
+++ b/xlators/features/qemu-block/src/bdrv-xlator.c
@@ -69,10 +69,13 @@ static int
 qemu_gluster_open (BlockDriverState *bs, QDict *options, int bdrv_flags)
 {
 	inode_t *inode = NULL;
-	BDRVGlusterState *s = NULL;
+	BDRVGlusterState *s = bs->opaque;
 	QemuOpts *opts = NULL;
 	Error *local_err = NULL;
 	const char *filename = NULL;
+	char gfid_str[128];
+	int ret;
+	qb_conf_t *conf = THIS->private;
 
 	opts = qemu_opts_create_nofail(&runtime_opts);
 	qemu_opts_absorb_qdict(opts, options, &local_err);
@@ -84,12 +87,40 @@ qemu_gluster_open (BlockDriverState *bs, QDict *options, int bdrv_flags)
 
 	filename = qemu_opt_get(opts, "filename");
 
-	inode = qb_inode_from_filename (filename);
-	if (!inode)
-		return -EINVAL;
+	/*
+	 * gfid:<gfid> format means we're opening a backing image.
+	 */
+	ret = sscanf(filename, "gluster://gfid:%s", gfid_str);
+	if (ret) {
+		loc_t loc = {0,};
+		struct iatt buf = {0,};
+		uuid_t gfid;
 
-	s = bs->opaque;
-	s->inode = inode_ref (inode);
+		uuid_parse(gfid_str, gfid);
+
+		loc.inode = inode_find(conf->root_inode->table, gfid);
+		if (!loc.inode) {
+			loc.inode = inode_new(conf->root_inode->table);
+			uuid_copy(loc.inode->gfid, gfid);
+		}
+
+		uuid_copy(loc.gfid, loc.inode->gfid);
+		ret = syncop_lookup(FIRST_CHILD(THIS), &loc, NULL, &buf, NULL,
+				    NULL);
+		if (ret) {
+			loc_wipe(&loc);
+			return -errno;
+		}
+
+		s->inode = inode_ref(loc.inode);
+		loc_wipe(&loc);
+	} else {
+		inode = qb_inode_from_filename (filename);
+		if (!inode)
+			return -EINVAL;
+
+		s->inode = inode_ref(inode);
+	}
 
 	return 0;
 }
diff --git a/xlators/features/qemu-block/src/coroutine-synctask.c b/xlators/features/qemu-block/src/coroutine-synctask.c
index c3538f60e..e43988a95 100644
--- a/xlators/features/qemu-block/src/coroutine-synctask.c
+++ b/xlators/features/qemu-block/src/coroutine-synctask.c
@@ -22,192 +22,95 @@
 #include "qemu-block-memory-types.h"
 
 #include "qemu-block.h"
-#include "coroutine-synctask.h"
 
-void
-qemu_coroutine_delete (Coroutine *co_)
-{
-	struct synctask *synctask = NULL;
-	CoroutineSynctask *cs = NULL;
-
-	cs = DO_UPCAST(CoroutineSynctask, base, co_);
-	synctask = cs->synctask;
-
-	cs->die = 1;
-	synctask_wake (synctask);
-
-	/* Do not free either @cs or @synctask here.
-	   @synctask is naturally destroyed when
-	   cs_proc() returns (after "break"ing out of
-	   the loop because of setting cs->die=1 above.
-
-	   We free @cs too just before returning from
-	   cs_proc()
-	*/
-	return;
-}
-
-
-CoroutineAction
-qemu_coroutine_switch (Coroutine *from_, Coroutine *to_, CoroutineAction action)
+/*
+ * This code serves as the bridge from the main glusterfs context to the qemu
+ * coroutine context via synctask. We create a single threaded syncenv with a
+ * single synctask responsible for processing a queue of coroutines. The qemu
+ * code invoked from within the synctask function handlers uses the ucontext
+ * coroutine implementation and scheduling logic internal to qemu. This
+ * effectively donates a thread of execution to qemu and its internal coroutine
+ * management.
+ *
+ * NOTE: The existence of concurrent synctasks has proven quite racy with regard
+ * to qemu coroutine management, particularly related to the lifecycle
+ * differences with top-level synctasks and internally created coroutines and
+ * interactions with qemu-internal queues (and locks, in turn). We explicitly
+ * disallow this scenario, via the queue, until it is more well supported.
+ */
+
+static struct {
+	struct list_head	queue;
+	gf_lock_t		lock;
+	struct synctask		*task;
+} qb_co;
+
+static void
+init_qbco()
 {
-	struct synctask *to = NULL;
-	struct synctask *from = NULL;
-	CoroutineSynctask *csto = NULL;
-	CoroutineSynctask *csfrom = NULL;
-
-	csto = DO_UPCAST(CoroutineSynctask, base, to_);
-	csfrom = DO_UPCAST(CoroutineSynctask, base, from_);
-	to = csto->synctask;
-	from = csfrom->synctask;
-
-	/* TODO: need mutex/cond guarding when making syncenv
-	   multithreaded
-	*/
-	csfrom->run = false;
-	csto->run = true;
-
-	/* the next three lines must be in this specific order only */
-	csfrom->action = action;
-
-	synctask_wake (to);
-
-	synctask_yield (from);
-
-	/* the yielder set @action value in @csfrom, but for the
-	   resumer it is @csto
-	*/
-	return csto->action;
+	INIT_LIST_HEAD(&qb_co.queue);
+	LOCK_INIT(&qb_co.lock);
 }
 
-
-int
-cs_fin (int ret, call_frame_t *frame, void *opaque)
+static int
+synctask_nop_cbk (int ret, call_frame_t *frame, void *opaque)
 {
-	/* nop */
 	return 0;
 }
 
-
 static int
-cs_proc (void *opaque)
+qb_synctask_wrap (void *opaque)
 {
-	CoroutineSynctask *cs = opaque;
-	struct synctask *synctask = NULL;
-
-	synctask = synctask_get (); /* == cs->synctask */
+	qb_local_t *qb_local, *tmp;
 
-	for (;;) {
-		while (!cs->run && !cs->die)
-			/* entry function (i.e cs->base.entry) will
-			   not be set just yet first time. Wait for
-			   caller to set it and call switch()
-			*/
-			synctask_yield (synctask);
+	LOCK(&qb_co.lock);
 
-		if (cs->die)
+	while (!list_empty(&qb_co.queue)) {
+		list_for_each_entry_safe(qb_local, tmp, &qb_co.queue, list) {
+			list_del_init(&qb_local->list);
 			break;
+		}
 
-		cs->base.entry (cs->base.entry_arg);
-		qemu_coroutine_switch (&cs->base, cs->base.caller,
-				       COROUTINE_TERMINATE);
-	}
-
-	GF_FREE (cs);
-
-	return 0;
-}
-
-
-Coroutine *
-qemu_coroutine_new()
-{
-	qb_conf_t *conf = NULL;
-	CoroutineSynctask *cs = NULL;
-	struct synctask *task = NULL;
-
-	conf = THIS->private;
-
-	cs = GF_CALLOC (1, sizeof (*cs), gf_qb_mt_coroutinesynctask_t);
-	if (!cs)
-		return NULL;
-
-	task = synctask_get ();
-	/* Inherit the frame from the parent synctask, as this will
-	   carry forward things like uid, gid, pid, lkowner etc. of the
-	   caller properly.
-	*/
-	cs->synctask = synctask_create (conf->env, cs_proc, cs_fin,
-					task ? task->frame : NULL, cs);
-	if (!cs->synctask)
-		return NULL;
-
-	return &cs->base;
-}
-
+		UNLOCK(&qb_co.lock);
 
-Coroutine *
-qemu_coroutine_self()
-{
-	struct synctask *synctask = NULL;
-	CoroutineSynctask *cs = NULL;
-
-	synctask = synctask_get();
-
-	cs = synctask->opaque;
-
-	return &cs->base;
-}
-
-
-bool
-qemu_in_coroutine ()
-{
-	Coroutine *co = NULL;
+		qb_local->synctask_fn(qb_local);
+		/* qb_local is now unwound and gone! */
 
-	co = qemu_coroutine_self ();
-
-	return co && co->caller;
-}
+		LOCK(&qb_co.lock);
+	}
 
+	qb_co.task = NULL;
 
-/* These are calls for the "top" xlator to invoke/submit
-   coroutines
-*/
+	UNLOCK(&qb_co.lock);
 
-static int
-synctask_nop_cbk (int ret, call_frame_t *frame, void *opaque)
-{
 	return 0;
 }
 
-
-int
-qb_synctask_wrap (void *opaque)
-{
-	struct synctask *task = NULL;
-	CoroutineSynctask *cs = NULL;
-	qb_local_t *qb_local = NULL;
-
-	task = synctask_get ();
-	cs = opaque;
-	cs->synctask = task;
-	qb_local = DO_UPCAST (qb_local_t, cs, cs);
-
-	return qb_local->synctask_fn (opaque);
-}
-
-
 int
 qb_coroutine (call_frame_t *frame, synctask_fn_t fn)
 {
 	qb_local_t *qb_local = NULL;
 	qb_conf_t *qb_conf = NULL;
+	static int init = 0;
 
 	qb_local = frame->local;
 	qb_local->synctask_fn = fn;
 	qb_conf = frame->this->private;
 
-	return synctask_new (qb_conf->env, qb_synctask_wrap, synctask_nop_cbk,
-			     frame, &qb_local->cs);
+	if (!init) {
+		init = 1;
+		init_qbco();
+	}
+
+	LOCK(&qb_co.lock);
+
+	if (!qb_co.task)
+		qb_co.task = synctask_create(qb_conf->env, qb_synctask_wrap,
+					     synctask_nop_cbk, frame, NULL);
+
+	list_add_tail(&qb_local->list, &qb_co.queue);
+
+	UNLOCK(&qb_co.lock);
+
+	return 0;
 }
diff --git a/xlators/features/qemu-block/src/coroutine-synctask.h b/xlators/features/qemu-block/src/coroutine-synctask.h
deleted file mode 100644
index 0f0aa99cc..000000000
--- a/xlators/features/qemu-block/src/coroutine-synctask.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __COROUTINE_SYNCTASK_H
-#define __COROUTINE_SYNCTASK_H
-
-#include "syncop.h"
-#include "block/coroutine_int.h"
-#include "qemu-common.h"
-#include "block/coroutine_int.h"
-
-/*
-  Three entities:
-
-  synctask - glusterfs implementation of xlator friendly lightweight threads
-  Coroutine - qemu coroutine API for its block drivers
-  CoroutineSynctask - implementation of Coroutine using synctasks
-
-  Coroutine is an "embedded" structure inside CoroutineSynctask, called "base".
-
-  E.g:
-
-  Coroutine *co;
-  CoroutineSynctask *cs;
-  struct synctask *synctask;
-
-  cs == synctask->opaque;
-  co == &(cs->base);
-  cs = DO_UPCAST(CoroutineSynctask, base, co);
-  synctask == cs->synctask;
-
-*/
-
-typedef struct {
-	Coroutine base;
-	struct synctask *synctask;
-	CoroutineAction action;
-	bool run;
-	bool die;
-} CoroutineSynctask;
-
-
-
-#endif /* !__COROUTINE_SYNCTASK_H */
diff --git a/xlators/features/qemu-block/src/qb-coroutines.c b/xlators/features/qemu-block/src/qb-coroutines.c
index 9e64fecaf..7c52adb21 100644
--- a/xlators/features/qemu-block/src/qb-coroutines.c
+++ b/xlators/features/qemu-block/src/qb-coroutines.c
@@ -29,12 +29,13 @@
 int
 qb_format_and_resume (void *opaque)
 {
-	CoroutineSynctask *cs = NULL;
 	qb_local_t *local = NULL;
 	call_frame_t *frame = NULL;
 	call_stub_t *stub = NULL;
 	inode_t *inode = NULL;
 	char filename[64];
+	char base_filename[128];
+	int use_base = 0;
 	qb_inode_t *qb_inode = NULL;
 	Error *local_err = NULL;
 	fd_t *fd = NULL;
@@ -42,9 +43,7 @@ qb_format_and_resume (void *opaque)
 	qb_conf_t *qb_conf = NULL;
 	int ret = -1;
 
-	cs = opaque;
-
-	local = DO_UPCAST(qb_local_t, cs, cs);
+	local = opaque;
 	frame = local->frame;
 	stub = local->stub;
 	inode = local->inode;
@@ -54,8 +53,62 @@ qb_format_and_resume (void *opaque)
 
 	qb_inode = qb_inode_ctx_get (frame->this, inode);
 
-	bdrv_img_create (filename, qb_inode->fmt, 0, 0,
-			 0, qb_inode->size, 0, &local_err, true);
+	/*
+	 * See if the caller specified a backing image.
+	 */
+	if (!uuid_is_null(qb_inode->backing_gfid) || qb_inode->backing_fname) {
+		loc_t loc = {0,};
+		char gfid_str[64];
+		struct iatt buf;
+
+		if (!uuid_is_null(qb_inode->backing_gfid)) {
+			loc.inode = inode_find(qb_conf->root_inode->table,
+					qb_inode->backing_gfid);
+			if (!loc.inode) {
+				loc.inode = inode_new(qb_conf->root_inode->table);
+				uuid_copy(loc.inode->gfid,
+					qb_inode->backing_gfid);
+			}
+			uuid_copy(loc.gfid, loc.inode->gfid);
+		} else if (qb_inode->backing_fname) {
+			loc.inode = inode_new(qb_conf->root_inode->table);
+			loc.name = qb_inode->backing_fname;
+			loc.parent = inode_parent(inode, NULL, NULL);
+			loc_path(&loc, loc.name);
+		}
+
+		/*
+		 * Lookup the backing image. Verify existence and/or get the
+		 * gfid if we don't already have it.
+		 */
+		ret = syncop_lookup(FIRST_CHILD(frame->this), &loc, NULL, &buf,
+				    NULL, NULL);
+		GF_FREE(qb_inode->backing_fname);
+		if (ret) {
+			loc_wipe(&loc);
+			ret = errno;
+			goto err;
+		}
+
+		uuid_copy(qb_inode->backing_gfid, buf.ia_gfid);
+		loc_wipe(&loc);
+
+		/*
+		 * We pass the filename of the backing image into the qemu block
+		 * subsystem as the associated gfid. This is embedded into the
+		 * clone image and passed along to the gluster bdrv backend when
+		 * the block subsystem needs to operate on the backing image on
+		 * behalf of the clone.
+		 */
+		uuid_unparse(qb_inode->backing_gfid, gfid_str);
+		snprintf(base_filename, sizeof(base_filename),
+			 "gluster://gfid:%s", gfid_str);
+		use_base = 1;
+	}
+
+	bdrv_img_create (filename, qb_inode->fmt,
+			 use_base ? base_filename : NULL, 0, 0, qb_inode->size,
+			 0, &local_err, true);
 
 	if (error_is_set (&local_err)) {
 		gf_log (frame->this->name, GF_LOG_ERROR, "%s",
@@ -113,6 +166,10 @@ qb_format_and_resume (void *opaque)
 	QB_STUB_UNWIND (stub, 0, 0);
 
 	return 0;
+
+err:
+	QB_STUB_UNWIND(stub, -1, ret);
+	return 0;
 }
 
 
@@ -164,16 +221,13 @@ err:
 int
 qb_co_open (void *opaque)
 {
-	CoroutineSynctask *cs = NULL;
 	qb_local_t *local = NULL;
 	call_frame_t *frame = NULL;
 	call_stub_t *stub = NULL;
 	inode_t *inode = NULL;
 	qb_inode_t *qb_inode = NULL;
 
-	cs = opaque;
-
-	local = DO_UPCAST(qb_local_t, cs, cs);
+	local = opaque;
 	frame = local->frame;
 	stub = local->stub;
 	inode = local->inode;
@@ -202,7 +256,6 @@ qb_co_open (void *opaque)
 int
 qb_co_writev (void *opaque)
 {
-	CoroutineSynctask *cs = NULL;
 	qb_local_t *local = NULL;
 	call_frame_t *frame = NULL;
 	call_stub_t *stub = NULL;
@@ -211,9 +264,7 @@ qb_co_writev (void *opaque)
 	QEMUIOVector qiov = {0, };
 	int ret = 0;
 
-	cs = opaque;
-
-	local = DO_UPCAST(qb_local_t, cs, cs);
+	local = opaque;
 	frame = local->frame;
 	stub = local->stub;
 	inode = local->inode;
@@ -249,7 +300,6 @@ qb_co_writev (void *opaque)
 int
 qb_co_readv (void *opaque)
 {
-	CoroutineSynctask *cs = NULL;
 	qb_local_t *local = NULL;
 	call_frame_t *frame = NULL;
 	call_stub_t *stub = NULL;
@@ -260,9 +310,7 @@ qb_co_readv (void *opaque)
 	struct iovec iov = {0, };
 	int ret = 0;
 
-	cs = opaque;
-
-	local = DO_UPCAST(qb_local_t, cs, cs);
+	local = opaque;
 	frame = local->frame;
 	stub = local->stub;
 	inode = local->inode;
@@ -331,7 +379,6 @@ qb_co_readv (void *opaque)
 int
 qb_co_fsync (void *opaque)
 {
-	CoroutineSynctask *cs = NULL;
 	qb_local_t *local = NULL;
 	call_frame_t *frame = NULL;
 	call_stub_t *stub = NULL;
@@ -339,9 +386,7 @@ qb_co_fsync (void *opaque)
 	qb_inode_t *qb_inode = NULL;
 	int ret = 0;
 
-	cs = opaque;
-
-	local = DO_UPCAST(qb_local_t, cs, cs);
+	local = opaque;
 	frame = local->frame;
 	stub = local->stub;
 	inode = local->inode;
@@ -401,7 +446,6 @@ qb_update_size_xattr (xlator_t *this, fd_t *fd, const char *fmt, off_t offset)
 int
 qb_co_truncate (void *opaque)
 {
-	CoroutineSynctask *cs = NULL;
 	qb_local_t *local = NULL;
 	call_frame_t *frame = NULL;
 	call_stub_t *stub = NULL;
@@ -412,9 +456,8 @@ qb_co_truncate (void *opaque)
 	xlator_t *this = NULL;
 
 	this = THIS;
-	cs = opaque;
 
-	local = DO_UPCAST(qb_local_t, cs, cs);
+	local = opaque;
 	frame = local->frame;
 	stub = local->stub;
 	inode = local->inode;
@@ -463,14 +506,13 @@ out:
 int
 qb_co_close (void *opaque)
 {
-	CoroutineSynctask *cs = NULL;
 	qb_local_t *local = NULL;
 	call_frame_t *frame = NULL;
 	inode_t *inode = NULL;
 	qb_inode_t *qb_inode = NULL;
 	BlockDriverState *bs = NULL;
 
-	local = DO_UPCAST(qb_local_t, cs, cs);
+	local = opaque;
 	inode = local->inode;
 
 	qb_inode = qb_inode_ctx_get (THIS, inode);
@@ -493,7 +535,6 @@ qb_co_close (void *opaque)
 int
 qb_snapshot_create (void *opaque)
 {
-	CoroutineSynctask *cs = NULL;
 	qb_local_t *local = NULL;
 	call_frame_t *frame = NULL;
 	call_stub_t *stub = NULL;
@@ -503,9 +544,7 @@ qb_snapshot_create (void *opaque)
 	struct timeval tv = {0, };
 	int ret = 0;
 
-	cs = opaque;
-
-	local = DO_UPCAST(qb_local_t, cs, cs);
+	local = opaque;
 	frame = local->frame;
 	stub = local->stub;
 	inode = local->inode;
@@ -544,7 +583,6 @@ qb_snapshot_create (void *opaque)
 int
 qb_snapshot_delete (void *opaque)
 {
-	CoroutineSynctask *cs = NULL;
 	qb_local_t *local = NULL;
 	call_frame_t *frame = NULL;
 	call_stub_t *stub = NULL;
@@ -552,9 +590,7 @@ qb_snapshot_delete (void *opaque)
 	qb_inode_t *qb_inode = NULL;
 	int ret = 0;
 
-	cs = opaque;
-
-	local = DO_UPCAST(qb_local_t, cs, cs);
+	local = opaque;
 	frame = local->frame;
 	stub = local->stub;
 	inode = local->inode;
@@ -588,7 +624,6 @@ qb_snapshot_delete (void *opaque)
 int
 qb_snapshot_goto (void *opaque)
 {
-	CoroutineSynctask *cs = NULL;
 	qb_local_t *local = NULL;
 	call_frame_t *frame = NULL;
 	call_stub_t *stub = NULL;
@@ -596,9 +631,7 @@ qb_snapshot_goto (void *opaque)
 	qb_inode_t *qb_inode = NULL;
 	int ret = 0;
 
-	cs = opaque;
-
-	local = DO_UPCAST(qb_local_t, cs, cs);
+	local = opaque;
 	frame = local->frame;
 	stub = local->stub;
 	inode = local->inode;
diff --git a/xlators/features/qemu-block/src/qb-coroutines.h b/xlators/features/qemu-block/src/qb-coroutines.h
index 6707d6d6c..583319f3b 100644
--- a/xlators/features/qemu-block/src/qb-coroutines.h
+++ b/xlators/features/qemu-block/src/qb-coroutines.h
@@ -15,7 +15,6 @@
 #include "call-stub.h"
 #include "block/block_int.h"
 #include "monitor/monitor.h"
-#include "coroutine-synctask.h"
 
 int qb_format_and_resume (void *opaque);
 int qb_snapshot_create (void *opaque);
diff --git a/xlators/features/qemu-block/src/qemu-block.c b/xlators/features/qemu-block/src/qemu-block.c
index c171f16f4..48bbf3140 100644
--- a/xlators/features/qemu-block/src/qemu-block.c
+++ b/xlators/features/qemu-block/src/qemu-block.c
@@ -108,42 +108,76 @@ qb_iatt_fixup (xlator_t *this, inode_t *inode, struct iatt *iatt)
 int
 qb_format_extract (xlator_t *this, char *format, inode_t *inode)
 {
-	char       *s = NULL;
+	char       *s, *save;
 	uint64_t    size = 0;
 	char        fmt[QB_XATTR_VAL_MAX+1] = {0, };
 	qb_inode_t *qb_inode = NULL;
+	char *formatstr = NULL;
+	uuid_t gfid = {0,};
+	char gfid_str[64] = {0,};
+	int ret;
 
-	strncpy (fmt, format, QB_XATTR_VAL_MAX);
-	s = strchr (fmt, ':');
+	strncpy(fmt, format, QB_XATTR_VAL_MAX);
+
+	s = strtok_r(fmt, ":", &save);
 	if (!s)
 		goto invalid;
-	if (s == fmt)
-		goto invalid;
+	formatstr = gf_strdup(s);
 
-	*s = 0; s++;
-	if (!*s || strchr (s, ':'))
+	s = strtok_r(NULL, ":", &save);
+	if (!s)
 		goto invalid;
-
 	if (gf_string2bytesize (s, &size))
 		goto invalid;
-
 	if (!size)
 		goto invalid;
 
+	s = strtok_r(NULL, "\0", &save);
+	if (s && !strncmp(s, "<gfid:", strlen("<gfid:"))) {
+		/*
+		 * Check for valid gfid backing image specifier.
+		 */
+		if (strlen(s) + 1 > sizeof(gfid_str))
+			goto invalid;
+		ret = sscanf(s, "<gfid:%[^>]s", gfid_str);
+		if (ret == 1) {
+			ret = uuid_parse(gfid_str, gfid);
+			if (ret < 0)
+				goto invalid;
+		}
+	}
+
 	qb_inode = qb_inode_ctx_get (this, inode);
 	if (!qb_inode)
 		qb_inode = GF_CALLOC (1, sizeof (*qb_inode),
 				      gf_qb_mt_qb_inode_t);
-	if (!qb_inode)
+	if (!qb_inode) {
+		GF_FREE(formatstr);
 		return ENOMEM;
+	}
 
-	strncpy (qb_inode->fmt, fmt, QB_XATTR_VAL_MAX);
+	strncpy(qb_inode->fmt, formatstr, QB_XATTR_VAL_MAX);
 	qb_inode->size = size;
-	qb_inode->size_str = s;
+
+	/*
+	 * If a backing gfid was not specified, interpret any remaining bytes
+	 * associated with a backing image as a filename local to the parent
+	 * directory. The format processing will validate further.
+	 */
+	if (!uuid_is_null(gfid))
+		uuid_copy(qb_inode->backing_gfid, gfid);
+	else if (s)
+		qb_inode->backing_fname = gf_strdup(s);
 
 	inode_ctx_set (inode, this, (void *)&qb_inode);
+
+	GF_FREE(formatstr);
+
 	return 0;
+
 invalid:
+	GF_FREE(formatstr);
+
 	gf_log (this->name, GF_LOG_WARNING,
 		"invalid format '%s' in inode %s", format,
 		uuid_utoa (inode->gfid));
@@ -170,6 +204,7 @@ qb_local_init (call_frame_t *frame)
 	qb_local = GF_CALLOC (1, sizeof (*qb_local), gf_qb_mt_qb_local_t);
 	if (!qb_local)
 		return -1;
+	INIT_LIST_HEAD(&qb_local->list);
 
 	qb_local->frame = frame;
 	frame->local = qb_local;
@@ -191,6 +226,15 @@ qb_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 	if (op_ret == -1)
 		goto out;
 
+	/*
+	 * Cache the root inode for dealing with backing images. The format
+	 * coroutine and the gluster qemu backend driver both use the root inode
+	 * table to verify and/or redirect I/O to the backing image via
+	 * anonymous fd's.
+	 */
+	if (!conf->root_inode && __is_root_gfid(inode->gfid))
+		conf->root_inode = inode_ref(inode);
+
 	if (!xdata)
 		goto out;
 
@@ -249,6 +293,7 @@ qb_setxattr_format (call_frame_t *frame, xlator_t *this, call_stub_t *stub,
 	int op_errno = 0;
 	qb_local_t *qb_local = NULL;
 	data_t *data = NULL;
+	qb_inode_t *qb_inode;
 
 	if (!(data = dict_get (xattr, "trusted.glusterfs.block-format"))) {
 		QB_STUB_RESUME (stub);
@@ -264,12 +309,15 @@ qb_setxattr_format (call_frame_t *frame, xlator_t *this, call_stub_t *stub,
 		QB_STUB_UNWIND (stub, -1, op_errno);
 		return 0;
 	}
+	qb_inode = qb_inode_ctx_get(this, inode);
 
 	qb_local = frame->local;
 
 	qb_local->stub = stub;
 	qb_local->inode = inode_ref (inode);
-	strncpy (qb_local->fmt, format, QB_XATTR_VAL_MAX);
+
+	snprintf(qb_local->fmt, QB_XATTR_VAL_MAX, "%s:%lu", qb_inode->fmt,
+		 qb_inode->size);
 
 	qb_coroutine (frame, qb_format_and_resume);
 
@@ -651,6 +699,63 @@ enomem:
 	return 0;
 }
 
+static int32_t
+qb_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+		dict_t *xdata)
+{
+	qb_conf_t *conf = this->private;
+	gf_dirent_t *entry;
+	char *format;
+
+	list_for_each_entry(entry, &entries->list, list) {
+		if (!entry->inode || !entry->dict)
+			continue;
+
+		format = NULL;
+		if (dict_get_str(entry->dict, conf->qb_xattr_key, &format))
+			continue;
+
+		if (!format) {
+			qb_inode_cleanup(this, entry->inode, 1);
+			continue;
+		}
+
+		if (qb_format_extract(this, format, entry->inode))
+			continue;
+
+		qb_iatt_fixup(this, entry->inode, &entry->d_stat);
+	}
+
+	STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
+	return 0;
+}
+
+static int32_t
+qb_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+	    off_t off, dict_t *xdata)
+{
+	qb_conf_t *conf = this->private;
+
+	xdata = xdata ? dict_ref(xdata) : dict_new();
+	if (!xdata)
+		goto enomem;
+
+	if (dict_set_int32 (xdata, conf->qb_xattr_key, 0))
+		goto enomem;
+
+	STACK_WIND(frame, qb_readdirp_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+
+	dict_unref(xdata);
+	return 0;
+
+enomem:
+	QB_STACK_UNWIND(readdirp, frame, -1, ENOMEM, NULL, NULL);
+	if (xdata)
+		dict_unref(xdata);
+	return 0;
+}
 
 int
 qb_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
@@ -925,6 +1030,7 @@ init (xlator_t *this)
 {
         qb_conf_t *conf    = NULL;
         int32_t    ret     = -1;
+	static int bdrv_inited = 0;
 
         if (!this->children || this->children->next) {
                 gf_log (this->name, GF_LOG_ERROR,
@@ -961,7 +1067,10 @@ init (xlator_t *this)
 
 	cur_mon = (void *) 1;
 
-	bdrv_init ();
+	if (!bdrv_inited) {
+		bdrv_init ();
+		bdrv_inited = 1;
+	}
 
 out:
         if (ret)
@@ -980,6 +1089,8 @@ fini (xlator_t *this)
 
         this->private = NULL;
 
+	if (conf->root_inode)
+		inode_unref(conf->root_inode);
         GF_FREE (conf);
 
 	return;
@@ -1005,6 +1116,7 @@ struct xlator_fops fops = {
 	.getxattr    = qb_getxattr,
 	.fgetxattr   = qb_fgetxattr
 */
+	.readdirp    = qb_readdirp,
 };
 
 
diff --git a/xlators/features/qemu-block/src/qemu-block.h b/xlators/features/qemu-block/src/qemu-block.h
index a91adb1ed..c95f2799a 100644
--- a/xlators/features/qemu-block/src/qemu-block.h
+++ b/xlators/features/qemu-block/src/qemu-block.h
@@ -15,7 +15,6 @@
 #include "call-stub.h"
 #include "block/block_int.h"
 #include "monitor/monitor.h"
-#include "coroutine-synctask.h"
 
 /* QB_XATTR_KEY_FMT is the on-disk xattr stored in the inode which
    indicates that the file must be "interpreted" by the block format
@@ -36,15 +35,16 @@
 
 #define QB_XATTR_KEY_MAX 64
 
-#define QB_XATTR_VAL_MAX 32
+#define QB_XATTR_VAL_MAX 64
 
 
 typedef struct qb_inode {
 	char fmt[QB_XATTR_VAL_MAX]; /* this is only the format, not "format:size" */
 	size_t size; /* virtual size in bytes */
-	char *size_str; /* pointer into fmt[] after ":" where size begins */
 	BlockDriverState *bs;
 	int refcnt;
+	uuid_t backing_gfid;
+	char *backing_fname;
 } qb_inode_t;
 
 
@@ -53,11 +53,11 @@ typedef struct qb_conf {
 	struct syncenv *env;
 	char qb_xattr_key[QB_XATTR_KEY_MAX];
 	char *default_password;
+	inode_t *root_inode;
 } qb_conf_t;
 
 
 typedef struct qb_local {
-	CoroutineSynctask  cs;
 	call_frame_t      *frame; /* backpointer */
 	call_stub_t       *stub;
 	inode_t           *inode;
@@ -65,12 +65,14 @@ typedef struct qb_local {
 	char               fmt[QB_XATTR_VAL_MAX+1];
 	char               name[256];
 	synctask_fn_t      synctask_fn;
+	struct list_head   list;
 } qb_local_t;
 
 void qb_local_free (xlator_t *this, qb_local_t *local);
 int qb_coroutine (call_frame_t *frame, synctask_fn_t fn);
 inode_t *qb_inode_from_filename (const char *filename);
 int qb_inode_to_filename (inode_t *inode, char *filename, int size);
+int qb_format_extract (xlator_t *this, char *format, inode_t *inode);
 
 qb_inode_t *qb_inode_ctx_get (xlator_t *this, inode_t *inode);
 
diff --git a/xlators/features/quiesce/src/quiesce.c b/xlators/features/quiesce/src/quiesce.c
index 73eb91947..24c7dc6ed 100644
--- a/xlators/features/quiesce/src/quiesce.c
+++ b/xlators/features/quiesce/src/quiesce.c
@@ -111,7 +111,7 @@ void
 gf_quiesce_enqueue (xlator_t *this, call_stub_t *stub)
 {
         quiesce_priv_t *priv    = NULL;
-        struct timeval  timeout = {0,};
+        struct timespec timeout = {0,};
 
         priv = this->private;
         if (!priv) {
@@ -129,7 +129,7 @@ gf_quiesce_enqueue (xlator_t *this, call_stub_t *stub)
 
         if (!priv->timer) {
                 timeout.tv_sec = 20;
-                timeout.tv_usec = 0;
+                timeout.tv_nsec = 0;
 
                 priv->timer = gf_timer_call_after (this->ctx,
                                                    timeout,
@@ -2492,7 +2492,7 @@ notify (xlator_t *this, int event, void *data, ...)
 {
         int             ret     = 0;
         quiesce_priv_t *priv    = NULL;
-        struct timeval  timeout = {0,};
+        struct timespec timeout = {0,};
 
         priv = this->private;
         if (!priv)
@@ -2525,7 +2525,7 @@ notify (xlator_t *this, int event, void *data, ...)
                 if (priv->timer)
                         break;
                 timeout.tv_sec = 20;
-                timeout.tv_usec = 0;
+                timeout.tv_nsec = 0;
 
                 priv->timer = gf_timer_call_after (this->ctx,
                                                    timeout,
diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am
index a6f49ae01..933c44019 100644
--- a/xlators/mgmt/glusterd/src/Makefile.am
+++ b/xlators/mgmt/glusterd/src/Makefile.am
@@ -11,7 +11,9 @@ glusterd_la_SOURCES = glusterd.c glusterd-handler.c glusterd-sm.c \
 	glusterd-volgen.c glusterd-rebalance.c glusterd-quota.c \
 	glusterd-geo-rep.c glusterd-replace-brick.c glusterd-log-ops.c \
 	glusterd-volume-ops.c glusterd-brick-ops.c glusterd-mountbroker.c \
-	glusterd-syncop.c glusterd-hooks.c glusterd-volume-set.c
+	glusterd-syncop.c glusterd-hooks.c glusterd-volume-set.c \
+        glusterd-locks.c glusterd-snapshot.c glusterd-mgmt-handler.c \
+        glusterd-mgmt.c
 
 glusterd_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 		     $(top_builddir)/rpc/xdr/src/libgfxdr.la \
@@ -21,7 +23,8 @@ glusterd_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 noinst_HEADERS = glusterd.h glusterd-utils.h glusterd-op-sm.h \
 	glusterd-sm.h glusterd-store.h glusterd-mem-types.h \
 	glusterd-pmap.h glusterd-volgen.h glusterd-mountbroker.h \
-	glusterd-syncop.h glusterd-hooks.h
+	glusterd-syncop.h glusterd-hooks.h glusterd-locks.h \
+	glusterd-mgmt.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(rpclibdir) -I$(CONTRIBDIR)/rbtree \
diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index c113516c8..596503c21 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -1025,6 +1025,8 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
         glusterd_brickinfo_t         *brickinfo      = NULL;
         glusterd_gsync_status_temp_t  param          = {0, };
         gf_boolean_t                  restart_needed = 0;
+        char                          msg[1024] __attribute__((unused)) = {0, };
+        int                           caps           = 0;
 
         GF_ASSERT (volinfo);
 
@@ -1105,12 +1107,31 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
 
         if (count)
                 brick = strtok_r (brick_list+1, " \n", &saveptr);
+#ifdef HAVE_BD_XLATOR
+        if (brickinfo->vg[0])
+                caps = CAPS_BD | CAPS_THIN |
+                        CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
+#endif
 
         while (i <= count) {
                 ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
                                                               &brickinfo);
                 if (ret)
                         goto out;
+#ifdef HAVE_BD_XLATOR
+                /* Check for VG/thin pool if its BD volume */
+                if (brickinfo->vg[0]) {
+                        ret = glusterd_is_valid_vg (brickinfo, 0, msg);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_CRITICAL, "%s", msg);
+                                goto out;
+                        }
+                        /* if anyone of the brick does not have thin support,
+                           disable it for entire volume */
+                        caps &= brickinfo->caps;
+                } else
+                        caps = 0;
+#endif
 
                 if (uuid_is_null (brickinfo->uuid)) {
                         ret = glusterd_resolve_brick (brickinfo);
@@ -1147,7 +1168,7 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
                 dict_foreach (volinfo->gsync_slaves,
                               _glusterd_restart_gsync_session, &param);
         }
-
+        volinfo->caps = caps;
 out:
         GF_FREE (free_ptr1);
         GF_FREE (free_ptr2);
@@ -1244,15 +1265,6 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        if (volinfo->backend == GD_VOL_BK_BD) {
-                snprintf (msg, sizeof (msg), "Add brick is not supported for "
-                           "Block backend volume %s.", volname);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                goto out;
-        }
-
         ret = glusterd_validate_volume_id (dict, volinfo);
         if (ret)
                 goto out;
@@ -1330,6 +1342,18 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
                 }
 
                 if (!uuid_compare (brickinfo->uuid, MY_UUID)) {
+#ifdef HAVE_BD_XLATOR
+                        if (brickinfo->vg[0]) {
+                                ret = glusterd_is_valid_vg (brickinfo, 1, msg);
+                                if (ret) {
+                                        gf_log (THIS->name, GF_LOG_ERROR, "%s",
+                                                msg);
+                                        *op_errstr = gf_strdup (msg);
+                                        goto out;
+                                }
+                        }
+#endif
+
                         ret = glusterd_validate_and_create_brickpath (brickinfo,
                                                           volinfo->volume_id,
                                                           op_errstr, is_force);
@@ -1364,7 +1388,6 @@ glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
         glusterd_volinfo_t *volinfo     = NULL;
         char               *errstr      = NULL;
         int32_t             brick_count = 0;
-        int32_t             replica_cnt = 0;
         char                msg[2048]   = {0,};
         int32_t             flag        = 0;
         gf1_op_commands     cmd         = GF_OP_CMD_NONE;
@@ -1421,10 +1444,10 @@ glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
         case GF_OP_CMD_START:
         {
                 if ((volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
-                    !dict_get_int32 (dict, "replica-count", &replica_cnt)) {
-                        snprintf (msg, sizeof(msg), "Rebalancing not needed "
-                                  "when reducing replica count. Try without "
-                                  "the 'start' option");
+                    dict_get (dict, "replica-count")) {
+                        snprintf (msg, sizeof(msg), "Migration of data is not "
+                                  "needed when reducing replica count. Use the"
+                                  " 'force' option");
                         errstr = gf_strdup (msg);
                         gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
                         goto out;
@@ -1457,7 +1480,7 @@ glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
                         goto out;
                 }
 
-                if (is_origin_glusterd ()) {
+                if (is_origin_glusterd (dict)) {
                         ret = glusterd_generate_and_set_task_id
                                 (dict, GF_REMOVE_BRICK_TID_KEY);
                         if (ret) {
@@ -1644,15 +1667,6 @@ glusterd_op_add_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        /* Need to reset the defrag/rebalance status accordingly */
-        switch (volinfo->rebal.defrag_status) {
-        case GF_DEFRAG_STATUS_FAILED:
-        case GF_DEFRAG_STATUS_COMPLETE:
-                volinfo->rebal.defrag_status = 0;
-        default:
-                break;
-        }
-
         ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
         if (ret)
                 goto out;
@@ -1684,6 +1698,8 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
         glusterd_brickinfo_t    *tmp           = NULL;
         char                    *task_id_str   = NULL;
         xlator_t                *this          = NULL;
+        dict_t                  *bricks_dict   = NULL;
+        char                    *brick_tmpstr  = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -1711,7 +1727,7 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
         /* Set task-id, if available, in ctx dict for operations other than
          * start
          */
-        if (is_origin_glusterd () && (cmd != GF_OP_CMD_START)) {
+        if (is_origin_glusterd (dict) && (cmd != GF_OP_CMD_START)) {
                 if (!uuid_is_null (volinfo->rebal.rebalance_id)) {
                         ret = glusterd_copy_uuid_to_dict
                                 (volinfo->rebal.rebalance_id, dict,
@@ -1724,10 +1740,13 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                 }
         }
 
-        /* Clear task-id & rebal.op on commmitting/stopping remove-brick */
+        /* Clear task-id, rebal.op and stored bricks on commmitting/stopping
+         * remove-brick */
         if ((cmd != GF_OP_CMD_START) || (cmd != GF_OP_CMD_STATUS)) {
                 uuid_clear (volinfo->rebal.rebalance_id);
                 volinfo->rebal.op = GD_OP_NONE;
+                dict_unref (volinfo->rebal.dict);
+                volinfo->rebal.dict = NULL;
         }
 
         ret = -1;
@@ -1812,7 +1831,20 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-
+        /* Save the list of bricks for later usage. Right now this is required
+         * for displaying the task parameters with task status in volume status.
+         */
+        bricks_dict = dict_new ();
+        if (!bricks_dict) {
+                ret = -1;
+                goto out;
+        }
+        ret = dict_set_int32 (bricks_dict, "count", count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to save remove-brick count");
+                goto out;
+        }
         while ( i <= count) {
                 snprintf (key, 256, "brick%d", i);
                 ret = dict_get_str (dict, key, &brick);
@@ -1822,6 +1854,21 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                         goto out;
                 }
 
+                brick_tmpstr = gf_strdup (brick);
+                if (!brick_tmpstr) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to duplicate brick name");
+                        goto out;
+                }
+                ret = dict_set_dynstr (bricks_dict, key, brick_tmpstr);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add brick to dict");
+                        goto out;
+                }
+                brick_tmpstr = NULL;
+
                 ret = glusterd_op_perform_remove_brick (volinfo, brick, force,
                                                         &need_rebalance);
                 if (ret)
@@ -1835,6 +1882,7 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                         volinfo->replica_count, replica_count,
                         volinfo->volname);
                 volinfo->replica_count = replica_count;
+                volinfo->sub_count = replica_count;
                 volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
                 volinfo->subvol_count = (volinfo->brick_count /
                                          volinfo->dist_leaf_count);
@@ -1851,6 +1899,8 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                         }
                 }
         }
+        volinfo->rebal.dict = bricks_dict;
+        bricks_dict = NULL;
 
         ret = glusterd_create_volfiles_and_notify_services (volinfo);
         if (ret) {
@@ -1895,5 +1945,9 @@ out:
         if (ret && err_str[0] && op_errstr)
                 *op_errstr = gf_strdup (err_str);
 
+        GF_FREE (brick_tmpstr);
+        if (bricks_dict)
+                dict_unref (bricks_dict);
+
         return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
index 055999898..5786694bd 100644
--- a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
+++ b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
@@ -39,6 +39,26 @@ glusterd_get_slave_info (char *slave, char **slave_ip,
 static int
 glusterd_gsync_read_frm_status (char *path, char *buf, size_t blen);
 
+struct gsync_config_opt_vals_ gsync_confopt_vals[] = {
+        {.op_name        = "change_detector",
+         .no_of_pos_vals = 2,
+         .case_sensitive = _gf_true,
+         .values         = {"xsync", "changelog"},
+        },
+        {.op_name        = "special_sync_mode",
+         .no_of_pos_vals = 2,
+         .case_sensitive = _gf_true,
+         .values         = {"partial", "recover"}
+        },
+        {.op_name        = "log-level",
+         .no_of_pos_vals = 5,
+         .case_sensitive = _gf_false,
+         .values         = {"critical", "error", "warning", "info", "debug"}
+        },
+        {.op_name = NULL,
+        },
+};
+
 static char *gsync_reserved_opts[] = {
         "gluster-command-dir",
         "pid-file",
@@ -782,15 +802,19 @@ out:
 static int
 gsync_verify_config_options (dict_t *dict, char **op_errstr, char *volname)
 {
-        char  **resopt    = NULL;
-        int     i         = 0;
-        int     ret       = -1;
-        char   *subop     = NULL;
-        char   *slave     = NULL;
-        char   *op_name   = NULL;
-        char   *op_value  = NULL;
-        char   *t         = NULL;
-        gf_boolean_t banned = _gf_true;
+        char                         **resopt           = NULL;
+        int                            i                = 0;
+        int                            ret              = -1;
+        char                          *subop            = NULL;
+        char                          *slave            = NULL;
+        char                          *op_name          = NULL;
+        char                          *op_value         = NULL;
+        char                          *t                = NULL;
+        char                           errmsg[PATH_MAX] = "";
+        gf_boolean_t                   banned           = _gf_true;
+        gf_boolean_t                   op_match         = _gf_true;
+        gf_boolean_t                   val_match        = _gf_true;
+        struct gsync_config_opt_vals_ *conf_vals        = NULL;
 
         if (dict_get_str (dict, "subop", &subop) != 0) {
                 gf_log ("", GF_LOG_WARNING, "missing subop");
@@ -864,6 +888,42 @@ gsync_verify_config_options (dict_t *dict, char **op_errstr, char *volname)
                 }
         }
 
+        /* Check options in gsync_confopt_vals for invalid values */
+        for (conf_vals = gsync_confopt_vals; conf_vals->op_name; conf_vals++) {
+                op_match = _gf_true;
+                for (i = 0; conf_vals->op_name[i] && op_name[i]; i++) {
+                        if (conf_vals->op_name[i] == op_name[i] ||
+                            (conf_vals->op_name[i] == '_' && op_name[i] == '-'))
+                                continue;
+                        op_match = _gf_false;
+                }
+
+                if (op_match) {
+                        val_match = _gf_false;
+                        for (i = 0; i < conf_vals->no_of_pos_vals; i++) {
+                                if(conf_vals->case_sensitive){
+                                        if (!strcmp (conf_vals->values[i], op_value))
+                                                val_match = _gf_true;
+                                } else {
+                                        if (!strcasecmp (conf_vals->values[i], op_value))
+                                                val_match = _gf_true;
+                                }
+                        }
+
+                        if (!val_match) {
+                                ret = snprintf (errmsg, sizeof(errmsg) - 1,
+                                                "Invalid values (%s) for"
+                                                " option %s", op_value,
+                                                op_name);
+                                errmsg[ret] = '\0';
+
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                *op_errstr = gf_strdup (errmsg);
+                                return -1;
+                        }
+                }
+        }
+
         return 0;
 }
 
@@ -880,7 +940,7 @@ _get_status_mst_slv (dict_t *this, char *key, data_t *value, void *data)
         char                          *slave_buf = NULL;
         char                          *slave_ip  = NULL;
         char                          *slave_vol = NULL;
-        char                         **errmsg    = NULL;
+        char                          *errmsg    = NULL;
         char                           conf_path[PATH_MAX] = "";
         int                           ret = -1;
         glusterd_conf_t              *priv = NULL;
@@ -902,10 +962,14 @@ _get_status_mst_slv (dict_t *this, char *key, data_t *value, void *data)
                 return 0;
         slave++;
 
-        ret = glusterd_get_slave_info (slave, &slave_ip, &slave_vol, errmsg);
+        ret = glusterd_get_slave_info (slave, &slave_ip, &slave_vol, &errmsg);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to fetch slave details.");
+                if (errmsg)
+                        gf_log ("", GF_LOG_ERROR, "Unable to fetch "
+                                "slave details. Error: %s", errmsg);
+                else
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to fetch slave details.");
                 ret = -1;
                 goto out;
         }
@@ -1719,7 +1783,6 @@ glusterd_mountbroker_check (char **slave_ip, char **op_errstr)
 
         GF_ASSERT (slave_ip);
         GF_ASSERT (*slave_ip);
-        GF_ASSERT (op_errstr);
 
         /* Checking if hostname has user specified */
         host = strstr (*slave_ip, "@");
@@ -1738,7 +1801,8 @@ glusterd_mountbroker_check (char **slave_ip, char **op_errstr)
                         errmsg[ret] = '\0';
                         gf_log ("", GF_LOG_ERROR, "%s", errmsg);
                         ret = -1;
-                        *op_errstr = gf_strdup (errmsg);
+                        if (op_errstr)
+                                *op_errstr = gf_strdup (errmsg);
                         goto out;
                 }
 
@@ -1751,7 +1815,8 @@ glusterd_mountbroker_check (char **slave_ip, char **op_errstr)
                                         "Non-root username (%s@%s) not allowed.",
                                         username, tmp);
                         errmsg[ret] = '\0';
-                        *op_errstr = gf_strdup (errmsg);
+                        if (op_errstr)
+                                *op_errstr = gf_strdup (errmsg);
                         gf_log ("", GF_LOG_ERROR,
                                 "Non-Root username not allowed.");
                         ret = -1;
@@ -2042,6 +2107,8 @@ glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr)
         if (ret)
                 goto out;
 
+        uuid_utoa_r (MY_UUID, uuid_str);
+
         if (conf->op_version < 2) {
                 gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
                 snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index afc223669..71d076624 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -33,6 +33,7 @@
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "glusterd-store.h"
+#include "glusterd-locks.h"
 
 #include "glusterd1-xdr.h"
 #include "cli1-xdr.h"
@@ -54,6 +55,8 @@
 #include <lvm2app.h>
 #endif
 
+extern uuid_t global_txn_id;
+
 int glusterd_big_locked_notify (struct rpc_clnt *rpc, void *mydata,
                                 rpc_clnt_event_t event,
                                 void *data, rpc_clnt_notify_t notify_fn)
@@ -335,7 +338,9 @@ glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
         char                    *volume_id_str  = NULL;
         struct args_pack        pack = {0,};
         xlator_t                *this = NULL;
-
+#ifdef HAVE_BD_XLATOR
+        int                     caps = 0;
+#endif
 
         GF_ASSERT (volinfo);
         GF_ASSERT (volumes);
@@ -360,6 +365,21 @@ glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
         if (ret)
                 goto out;
 
+        /* As of now, the snap volumes are also displayed as part of
+           volume info command. So this change is to display whether
+           the volume is original volume or the snap_volume. If
+           displaying of snap volumes in volume info o/p is not needed
+           this should be removed.
+        */
+        snprintf (key, 256, "volume%d.snap_volume", count);
+        ret = dict_set_int32 (volumes, key, volinfo->is_snap_volume);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "failed to set whether "
+                        "the volume is a snap volume or actual volume (%s)",
+                        volinfo->volname);
+                goto out;
+        }
+
         snprintf (key, 256, "volume%d.brick_count", count);
         ret = dict_set_int32 (volumes, key, volinfo->brick_count);
         if (ret)
@@ -400,10 +420,71 @@ glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
                 goto out;
 
 #ifdef HAVE_BD_XLATOR
-        snprintf (key, 256, "volume%d.backend", count);
-        ret = dict_set_int32 (volumes, key, volinfo->backend);
-        if (ret)
-                goto out;
+        if (volinfo->caps) {
+                caps = 0;
+                snprintf (key, 256, "volume%d.xlator0", count);
+                buf = GF_MALLOC (256, gf_common_mt_char);
+                if (!buf) {
+                        ret = ENOMEM;
+                        goto out;
+                }
+                if (volinfo->caps & CAPS_BD)
+                        snprintf (buf, 256, "BD");
+                ret = dict_set_dynstr (volumes, key, buf);
+                if (ret) {
+                        GF_FREE (buf);
+                        goto out;
+                }
+
+                if (volinfo->caps & CAPS_THIN) {
+                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
+                                  caps++);
+                        buf = GF_MALLOC (256, gf_common_mt_char);
+                        if (!buf) {
+                                ret = ENOMEM;
+                                goto out;
+                        }
+                        snprintf (buf, 256, "thin");
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret) {
+                                GF_FREE (buf);
+                                goto out;
+                        }
+                }
+
+                if (volinfo->caps & CAPS_OFFLOAD_COPY) {
+                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
+                                  caps++);
+                        buf = GF_MALLOC (256, gf_common_mt_char);
+                        if (!buf) {
+                                ret = ENOMEM;
+                                goto out;
+                        }
+                        snprintf (buf, 256, "offload_copy");
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret) {
+                                GF_FREE (buf);
+                                goto out;
+                        }
+                }
+
+                if (volinfo->caps & CAPS_OFFLOAD_SNAPSHOT) {
+                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
+                                  caps++);
+                        buf = GF_MALLOC (256, gf_common_mt_char);
+                        if (!buf) {
+                                ret = ENOMEM;
+                                goto out;
+                        }
+                        snprintf (buf, 256, "offload_snapshot");
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret)  {
+                                GF_FREE (buf);
+                                goto out;
+                        }
+                }
+
+        }
 #endif
 
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
@@ -425,6 +506,16 @@ glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
                 if (ret)
                         goto out;
 
+#ifdef HAVE_BD_XLATOR
+                if (volinfo->caps & CAPS_BD) {
+                        snprintf (key, 256, "volume%d.vg%d", count, i);
+                        snprintf (brick, 1024, "%s", brickinfo->vg);
+                        buf = gf_strdup (brick);
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret)
+                                goto out;
+                }
+#endif
                 i++;
         }
 
@@ -488,10 +579,17 @@ int32_t
 glusterd_op_txn_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
                        char *err_str, size_t err_len)
 {
-        int32_t                  ret    = -1;
-        xlator_t                *this   = NULL;
-        glusterd_conf_t         *priv   = NULL;
-        int32_t                  locked = 0;
+        int32_t                     ret             = -1;
+        dict_t                     *dict            = NULL;
+        xlator_t                   *this            = NULL;
+        glusterd_conf_t            *priv            = NULL;
+        int32_t                     locked          = 0;
+        char                       *tmp             = NULL;
+        char                       *volname         = NULL;
+        uuid_t                     *txn_id          = NULL;
+        uuid_t                     *originator_uuid = NULL;
+        glusterd_op_info_t          txn_op_info     = {{0},};
+        glusterd_op_sm_event_type_t event_type      = GD_OP_EVENT_NONE;
 
         GF_ASSERT (req);
         GF_ASSERT ((op > GD_OP_NONE) && (op < GD_OP_MAX));
@@ -502,33 +600,140 @@ glusterd_op_txn_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
         priv = this->private;
         GF_ASSERT (priv);
 
-        ret = glusterd_lock (MY_UUID);
+        dict = ctx;
+
+        /* Generate a transaction-id for this operation and
+         * save it in the dict. This transaction id distinguishes
+         * each transaction, and helps separate opinfos in the
+         * op state machine. */
+        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!txn_id)
+                goto out;
+
+        uuid_generate (*txn_id);
+
+        ret = dict_set_bin (dict, "transaction_id",
+                            txn_id, sizeof(*txn_id));
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to acquire lock on localhost, ret: %d", ret);
-                snprintf (err_str, err_len, "Another transaction is in progress. "
-                          "Please try again after sometime.");
+                       "Failed to set transaction id.");
                 goto out;
         }
 
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Transaction_id = %s", uuid_utoa (*txn_id));
+
+        /* Save the MY_UUID as the originator_uuid. This originator_uuid
+         * will be used by is_origin_glusterd() to determine if a node
+         * is the originator node for a command. */
+        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
+                                     gf_common_mt_uuid_t);
+        if (!originator_uuid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*originator_uuid, MY_UUID);
+        ret = dict_set_bin (dict, "originator_uuid",
+                            originator_uuid, sizeof (uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set originator uuid.");
+                goto out;
+        }
+
+        /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
+        if (priv->op_version < 3) {
+                ret = glusterd_lock (MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock on localhost, ret: %d",
+                                ret);
+                        snprintf (err_str, err_len,
+                                  "Another transaction is in progress. "
+                                  "Please try again after sometime.");
+                        goto out;
+                }
+        } else {
+                /* If no volname is given as a part of the command, locks will
+                 * not be held */
+                ret = dict_get_str (dict, "volname", &tmp);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Failed to get volume "
+                                "name");
+                        goto local_locking_done;
+                } else {
+                        /* Use a copy of volname, as cli response will be
+                         * sent before the unlock, and the volname in the
+                         * dict, might be removed */
+                        volname = gf_strdup (tmp);
+                        if (!volname)
+                                goto out;
+                }
+
+                ret = glusterd_mgmt_v3_lock (volname, MY_UUID, "vol");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock for %s", volname);
+                        snprintf (err_str, err_len,
+                                  "Another transaction is in progress for %s. "
+                                  "Please try again after sometime.", volname);
+                        goto out;
+                }
+        }
+
         locked = 1;
         gf_log (this->name, GF_LOG_DEBUG, "Acquired lock on localhost");
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_START_LOCK, NULL);
+local_locking_done:
+
+        /* If no volname is given as a part of the command, locks will
+         * not be held, hence sending stage event. */
+        if (volname)
+                event_type = GD_OP_EVENT_START_LOCK;
+        else {
+                txn_op_info.state.state = GD_OP_STATE_LOCK_SENT;
+                event_type = GD_OP_EVENT_ALL_ACC;
+        }
+
+        /* Save opinfo for this transaction with the transaction id */
+        glusterd_txn_opinfo_init (&txn_op_info, NULL, &op, ctx, req);
+
+        ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+                if (ctx)
+                        dict_unref (ctx);
+                goto out;
+        }
+
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, ctx);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Failed to acquire cluster"
                         " lock.");
                 goto out;
         }
 
-        glusterd_op_set_op (op);
-        glusterd_op_set_ctx (ctx);
-        glusterd_op_set_req (req);
-
-
 out:
-        if (locked && ret)
-                glusterd_unlock (MY_UUID);
+        if (locked && ret) {
+                /* Based on the op-version, we release the
+                 * cluster or mgmt_v3 lock */
+                if (priv->op_version < 3)
+                        glusterd_unlock (MY_UUID);
+                else {
+                        ret = glusterd_mgmt_v3_unlock (volname, MY_UUID,
+                                                       "vol");
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to release lock for %s",
+                                        volname);
+                        ret = -1;
+                }
+        }
+
+        if (volname)
+                GF_FREE (volname);
 
         gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
@@ -537,11 +742,15 @@ out:
 int
 __glusterd_handle_cluster_lock (rpcsvc_request_t *req)
 {
-        gd1_mgmt_cluster_lock_req       lock_req = {{0},};
-        int32_t                         ret = -1;
-        glusterd_op_lock_ctx_t          *ctx = NULL;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        xlator_t                        *this = NULL;
+        dict_t                         *op_ctx      = NULL;
+        int32_t                         ret         = -1;
+        gd1_mgmt_cluster_lock_req       lock_req    = {{0},};
+        glusterd_op_lock_ctx_t         *ctx         = NULL;
+        glusterd_op_t                   op          = GD_OP_EVENT_LOCK;
+        glusterd_peerinfo_t            *peerinfo    = NULL;
+        glusterd_op_info_t              txn_op_info = {{0},};
+        uuid_t                         *txn_id      = &global_txn_id;
+        xlator_t                       *this        = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -576,8 +785,29 @@ __glusterd_handle_cluster_lock (rpcsvc_request_t *req)
 
         uuid_copy (ctx->uuid, lock_req.uuid);
         ctx->req = req;
+        ctx->dict = NULL;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCK, ctx);
+        op_ctx =  dict_new ();
+        if (!op_ctx) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set new dict");
+                goto out;
+        }
+
+        glusterd_txn_opinfo_init (&txn_op_info, NULL, &op, op_ctx, req);
+
+        ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+                dict_unref (txn_op_info.op_ctx);
+                goto out;
+        }
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCK, txn_id, ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_LOCK");
 
 out:
         gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
@@ -653,6 +883,9 @@ __glusterd_handle_stage_op (rpcsvc_request_t *req)
         gd1_mgmt_stage_op_req           op_req = {{0},};
         glusterd_peerinfo_t             *peerinfo = NULL;
         xlator_t                        *this = NULL;
+        uuid_t                          *txn_id = &global_txn_id;
+        glusterd_op_info_t              txn_op_info = {{0},};
+        glusterd_op_sm_state_info_t     state;
 
         this = THIS;
         GF_ASSERT (this);
@@ -681,7 +914,36 @@ __glusterd_handle_stage_op (rpcsvc_request_t *req)
         if (ret)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_OP, req_ctx);
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
+        /* In cases where there is no volname, the receivers won't have a
+         * transaction opinfo created, as for those operations, the locking
+         * phase where the transaction opinfos are created, won't be called. */
+        ret = glusterd_get_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "No transaction's opinfo set");
+
+                state.state = GD_OP_STATE_LOCKED;
+                glusterd_txn_opinfo_init (&txn_op_info, &state,
+                                          &op_req.op, req_ctx->dict, req);
+
+                ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to set transaction's opinfo");
+                        dict_unref (req_ctx->dict);
+                        goto out;
+                }
+        }
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_OP,
+                                           txn_id, req_ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_STAGE_OP");
 
  out:
         free (op_req.buf.buf_val);//malloced by xdr
@@ -705,6 +967,7 @@ __glusterd_handle_commit_op (rpcsvc_request_t *req)
         gd1_mgmt_commit_op_req          op_req = {{0},};
         glusterd_peerinfo_t             *peerinfo = NULL;
         xlator_t                        *this = NULL;
+        uuid_t                          *txn_id = &global_txn_id;
 
         this = THIS;
         GF_ASSERT (this);
@@ -735,11 +998,12 @@ __glusterd_handle_commit_op (rpcsvc_request_t *req)
         if (ret)
                 goto out;
 
-        ret = glusterd_op_init_ctx (op_req.op);
-        if (ret)
-                goto out;
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_OP, req_ctx);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_OP,
+                                           txn_id, req_ctx);
 
 out:
         free (op_req.buf.buf_val);//malloced by xdr
@@ -1108,79 +1372,6 @@ glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
                                             __glusterd_handle_cli_get_volume);
 }
 
-#ifdef HAVE_BD_XLATOR
-int
-__glusterd_handle_cli_bd_op (rpcsvc_request_t *req)
-{
-        int32_t          ret        = -1;
-        gf_cli_req       cli_req    = { {0,} };
-        dict_t           *dict      = NULL;
-        char             *volname   = NULL;
-        char             op_errstr[2048] = {0,};
-        glusterd_op_t    cli_op     = GD_OP_BD_OP;
-
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                /* failed to decode msg */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_log ("glusterd", GF_LOG_DEBUG, "Received bd op req");
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                                "failed to get volname");
-                goto out;
-        }
-
-        ret = glusterd_op_begin (req, GD_OP_BD_OP, dict, op_errstr,
-                                 sizeof (op_errstr));
-out:
-        if (ret && dict)
-                dict_unref (dict);
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
-        if (ret) {
-                if (op_errstr[0] == '\0')
-                        snprintf (op_errstr, sizeof (op_errstr),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0,
-                                req, NULL, op_errstr);
-        }
-
-        return ret;
-}
-
-int
-glusterd_handle_cli_bd_op (rpcsvc_request_t *req)
-{
-        return glusterd_big_locked_handler (req, __glusterd_handle_cli_bd_op);
-}
-#endif
-
 int
 __glusterd_handle_cli_uuid_reset (rpcsvc_request_t *req)
 {
@@ -1863,6 +2054,57 @@ glusterd_op_unlock_send_resp (rpcsvc_request_t *req, int32_t status)
 }
 
 int
+glusterd_op_mgmt_v3_lock_send_resp (rpcsvc_request_t *req, uuid_t *txn_id,
+                                    int32_t status)
+{
+
+        gd1_mgmt_v3_lock_rsp           rsp = {{0},};
+        int                            ret = -1;
+
+        GF_ASSERT (req);
+        GF_ASSERT (txn_id);
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op_ret = status;
+        if (rsp.op_ret)
+               rsp.op_errno = errno;
+        uuid_copy (rsp.txn_id, *txn_id);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to mgmt_v3 lock, ret: %d",
+                ret);
+
+        return ret;
+}
+
+int
+glusterd_op_mgmt_v3_unlock_send_resp (rpcsvc_request_t *req, uuid_t *txn_id,
+                                      int32_t status)
+{
+
+        gd1_mgmt_v3_unlock_rsp          rsp = {{0},};
+        int                             ret = -1;
+
+        GF_ASSERT (req);
+        GF_ASSERT (txn_id);
+        rsp.op_ret = status;
+        if (rsp.op_ret)
+               rsp.op_errno = errno;
+        glusterd_get_uuid (&rsp.uuid);
+        uuid_copy (rsp.txn_id, *txn_id);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+
+        gf_log (THIS->name, GF_LOG_DEBUG,
+                "Responded to mgmt_v3 unlock, ret: %d",
+                ret);
+
+        return ret;
+}
+
+int
 __glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
 {
         gd1_mgmt_cluster_unlock_req     unlock_req = {{0}, };
@@ -1870,6 +2112,7 @@ __glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
         glusterd_op_lock_ctx_t          *ctx = NULL;
         glusterd_peerinfo_t             *peerinfo = NULL;
         xlator_t                        *this = NULL;
+        uuid_t                          *txn_id = &global_txn_id;
 
         this = THIS;
         GF_ASSERT (this);
@@ -1904,8 +2147,9 @@ __glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
         }
         uuid_copy (ctx->uuid, unlock_req.uuid);
         ctx->req = req;
+        ctx->dict = NULL;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_UNLOCK, ctx);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_UNLOCK, txn_id, ctx);
 
 out:
         glusterd_friend_sm ();
@@ -3059,9 +3303,13 @@ glusterd_xfer_friend_add_resp (rpcsvc_request_t *req, char *myhostname,
 }
 
 static void
-get_probe_error_str (int op_ret, int op_errno, char *errstr, size_t len,
-                     char *hostname, int port)
+set_probe_error_str (int op_ret, int op_errno, char *op_errstr, char *errstr,
+                     size_t len, char *hostname, int port)
 {
+        if ((op_errstr) && (strcmp (op_errstr, ""))) {
+                snprintf (errstr, len, "%s", op_errstr);
+                return;
+        }
 
         if (!op_ret) {
                 switch (op_errno) {
@@ -3141,11 +3389,8 @@ glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
         GF_ASSERT (req);
         GF_ASSERT (this);
 
-        if (op_errstr == NULL)
-                (void) get_probe_error_str (op_ret, op_errno, errstr,
-                                            sizeof (errstr), hostname, port);
-        else
-                snprintf (errstr, sizeof (errstr), "%s", op_errstr);
+        (void) set_probe_error_str (op_ret, op_errno, op_errstr, errstr,
+                                    sizeof (errstr), hostname, port);
 
         if (dict) {
                 ret = dict_get_str (dict, "cmd-str", &cmd_str);
@@ -3174,9 +3419,14 @@ glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
 }
 
 static void
-get_deprobe_error_str (int op_ret, int op_errno, char *errstr, size_t len,
-                       char *hostname)
+set_deprobe_error_str (int op_ret, int op_errno, char *op_errstr, char *errstr,
+                       size_t len, char *hostname)
 {
+        if ((op_errstr) && (strcmp (op_errstr, ""))) {
+                snprintf (errstr, len, "%s", op_errstr);
+                return;
+        }
+
         if (op_ret) {
                 switch (op_errno) {
                         case GF_DEPROBE_LOCALHOST:
@@ -3228,8 +3478,8 @@ glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
 
         GF_ASSERT (req);
 
-        (void) get_deprobe_error_str (op_ret, op_errno, errstr, sizeof (errstr),
-                                      hostname);
+        (void) set_deprobe_error_str (op_ret, op_errno, op_errstr, errstr,
+                                      sizeof (errstr), hostname);
 
         if (dict) {
                 ret = dict_get_str (dict, "cmd-str", &cmd_str);
@@ -3615,8 +3865,12 @@ get_brickinfo_from_brickid (char *brickid, glusterd_brickinfo_t **brickinfo)
 
         uuid_parse (volid_str, volid);
         ret = glusterd_volinfo_find_by_volume_id (volid, &volinfo);
-        if (ret)
-                goto out;
+        if (ret) {
+                /* Check if it a snapshot volume */
+                ret = glusterd_snap_volinfo_find_by_volume_id (volid, &volinfo);
+                if (ret)
+                        goto out;
+        }
 
         ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
                                                       brickinfo);
@@ -3790,6 +4044,7 @@ __glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
         glusterd_peerinfo_t  *peerinfo    = NULL;
         glusterd_peerctx_t   *peerctx     = NULL;
         gf_boolean_t         quorum_action = _gf_false;
+        glusterd_volinfo_t   *volinfo     = NULL;
 
         peerctx = mydata;
         if (!peerctx)
@@ -3817,6 +4072,20 @@ __glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_DISCONNECT %d",
                         peerinfo->state.state);
 
+                if (peerinfo->connected) {
+                        list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                                ret = glusterd_mgmt_v3_unlock (volinfo->volname,
+                                                               peerinfo->uuid,
+                                                               "vol");
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_TRACE,
+                                                "Lock not released for %s",
+                                                volinfo->volname);
+                        }
+
+                        ret = 0;
+                }
+
                 if ((peerinfo->quorum_contrib != QUORUM_DOWN) &&
                     (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED)) {
                         peerinfo->quorum_contrib = QUORUM_DOWN;
@@ -3932,11 +4201,9 @@ rpcsvc_actor_t gd_svc_cli_actors[] = {
         [GLUSTER_CLI_STATEDUMP_VOLUME]   = {"STATEDUMP_VOLUME",   GLUSTER_CLI_STATEDUMP_VOLUME, glusterd_handle_cli_statedump_volume,  NULL, 0, DRC_NA},
         [GLUSTER_CLI_LIST_VOLUME]        = {"LIST_VOLUME",        GLUSTER_CLI_LIST_VOLUME,      glusterd_handle_cli_list_volume,       NULL, 0, DRC_NA},
         [GLUSTER_CLI_CLRLOCKS_VOLUME]    = {"CLEARLOCKS_VOLUME",  GLUSTER_CLI_CLRLOCKS_VOLUME,  glusterd_handle_cli_clearlocks_volume, NULL, 0, DRC_NA},
-#ifdef HAVE_BD_XLATOR
-        [GLUSTER_CLI_BD_OP]              = {"BD_OP",              GLUSTER_CLI_BD_OP,            glusterd_handle_cli_bd_op,             NULL, 0, DRC_NA},
-#endif
-        [GLUSTER_CLI_COPY_FILE]     = {"COPY_FILE", GLUSTER_CLI_COPY_FILE, glusterd_handle_copy_file, NULL, 0, DRC_NA},
-        [GLUSTER_CLI_SYS_EXEC]      = {"SYS_EXEC", GLUSTER_CLI_SYS_EXEC, glusterd_handle_sys_exec, NULL, 0, DRC_NA},
+        [GLUSTER_CLI_COPY_FILE]          = {"COPY_FILE",          GLUSTER_CLI_COPY_FILE,        glusterd_handle_copy_file,             NULL, 0, DRC_NA},
+        [GLUSTER_CLI_SYS_EXEC]           = {"SYS_EXEC",           GLUSTER_CLI_SYS_EXEC,         glusterd_handle_sys_exec,              NULL, 0, DRC_NA},
+        [GLUSTER_CLI_SNAP]               = {"SNAP",               GLUSTER_CLI_SNAP,             glusterd_handle_snapshot,              NULL, 0, DRC_NA},
 };
 
 struct rpcsvc_program gd_svc_cli_prog = {
@@ -3947,3 +4214,24 @@ struct rpcsvc_program gd_svc_cli_prog = {
         .actors    = gd_svc_cli_actors,
 	.synctask  = _gf_true,
 };
+
+/* This is a minimal RPC prog, which contains only the readonly RPC procs from
+ * the cli rpcsvc
+ */
+rpcsvc_actor_t gd_svc_cli_actors_ro[] = {
+        [GLUSTER_CLI_LIST_FRIENDS]       = { "LIST_FRIENDS",      GLUSTER_CLI_LIST_FRIENDS,     glusterd_handle_cli_list_friends,      NULL, 0, DRC_NA},
+        [GLUSTER_CLI_UUID_GET]           = { "UUID_GET",          GLUSTER_CLI_UUID_GET,         glusterd_handle_cli_uuid_get,          NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GET_VOLUME]         = { "GET_VOLUME",        GLUSTER_CLI_GET_VOLUME,       glusterd_handle_cli_get_volume,        NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GETWD]              = { "GETWD",             GLUSTER_CLI_GETWD,            glusterd_handle_getwd,                 NULL, 1, DRC_NA},
+        [GLUSTER_CLI_STATUS_VOLUME]      = {"STATUS_VOLUME",      GLUSTER_CLI_STATUS_VOLUME,    glusterd_handle_status_volume,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_LIST_VOLUME]        = {"LIST_VOLUME",        GLUSTER_CLI_LIST_VOLUME,      glusterd_handle_cli_list_volume,       NULL, 0, DRC_NA},
+};
+
+struct rpcsvc_program gd_svc_cli_prog_ro = {
+        .progname  = "GlusterD svc cli read-only",
+        .prognum   = GLUSTER_CLI_PROGRAM,
+        .progver   = GLUSTER_CLI_VERSION,
+        .numactors = GLUSTER_CLI_MAXVALUE,
+        .actors    = gd_svc_cli_actors_ro,
+	.synctask  = _gf_true,
+};
diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c
index ab172f69a..0f0357c4c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handshake.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c
@@ -30,26 +30,147 @@
 
 extern struct rpc_clnt_program gd_peer_prog;
 extern struct rpc_clnt_program gd_mgmt_prog;
+extern struct rpc_clnt_program gd_mgmt_v3_prog;
+extern struct rpc_clnt_program gd_mgmt_v3_prog;
+
 
 #define TRUSTED_PREFIX         "trusted-"
 
 typedef ssize_t (*gfs_serialize_t) (struct iovec outmsg, void *data);
 
+static int
+get_snap_volname_and_volinfo (const char *volpath, char **volname,
+                              glusterd_volinfo_t **volinfo)
+{
+        int              ret            = -1;
+        char            *save_ptr       = NULL;
+        char            *str_token      = NULL;
+        char            *snapname       = NULL;
+        char            *volname_token  = NULL;
+        char            *vol            = NULL;
+        glusterd_snap_t *snap           = NULL;
+        xlator_t        *this           = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (volpath);
+        GF_ASSERT (volinfo);
+
+        str_token = gf_strdup (volpath);
+        if (NULL == str_token) {
+                goto out;
+        }
+
+        /* Input volname will have below formats:
+         * /snaps/<snapname>/<volname>.<hostname>
+         * or
+         * /snaps/<snapname>/<parent-volname>
+         * We need to extract snapname and parent_volname */
+
+        /*split string by "/" */
+        strtok_r (str_token, "/",  &save_ptr);
+        snapname  = strtok_r(NULL, "/", &save_ptr);
+        if (!snapname) {
+                gf_log(this->name, GF_LOG_ERROR, "Invalid path: %s", volpath);
+                goto out;
+        }
+
+        volname_token = strtok_r(NULL, "/", &save_ptr);
+        if (!volname_token) {
+                gf_log(this->name, GF_LOG_ERROR, "Invalid path: %s", volpath);
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (!snap) {
+                gf_log(this->name, GF_LOG_ERROR, "Failed to "
+                                "fetch snap %s", snapname);
+                goto out;
+        }
+
+        /* Find if its a parent volume name or snap volume
+         * name. This function will succeed if volname_token
+         * is a parent volname
+         */
+        ret = glusterd_volinfo_find (volname_token, volinfo);
+        if (ret) {
+                *volname = gf_strdup (volname_token);
+                if (NULL == *volname) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_snap_volinfo_find (volname_token, snap,
+                                                  volinfo);
+                if (ret) {
+                        /* Split the volume name */
+                        vol = strtok_r (volname_token, ".", &save_ptr);
+                        if (!vol) {
+                                gf_log(this->name, GF_LOG_ERROR, "Invalid "
+                                                "volname (%s)", volname_token);
+                                goto out;
+                        }
+
+                        ret = glusterd_snap_volinfo_find (vol, snap, volinfo);
+                        if (ret) {
+                                gf_log(this->name, GF_LOG_ERROR, "Failed to "
+                                       "fetch snap volume from volname (%s)",
+                                       vol);
+                                goto out;
+                        }
+                }
+        } else {
+                /*volname_token is parent volname*/
+                ret = glusterd_snap_volinfo_find_from_parent_volname (
+                                volname_token, snap, volinfo);
+                if (ret) {
+                        gf_log(this->name, GF_LOG_ERROR, "Failed to "
+                                        "fetch snap volume from parent "
+                                        "volname (%s)", volname_token);
+                        goto out;
+                }
+
+                /* Since volname_token is a parent volname we should
+                 * get the snap volname here*/
+                *volname = gf_strdup ((*volinfo)->volname);
+                if (NULL == *volname) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+out:
+        if (ret && NULL != *volname) {
+                GF_FREE (*volname);
+                *volname = NULL;
+        }
+        return ret;
+}
+
 static size_t
 build_volfile_path (const char *volname, char *path,
                     size_t path_len, char *trusted_str)
 {
-        struct stat         stbuf       = {0,};
-        int32_t             ret         = -1;
-        glusterd_conf_t    *priv        = NULL;
-        char               *vol         = NULL;
-        char               *dup_volname = NULL;
-        char               *free_ptr    = NULL;
-        char               *tmp         = NULL;
-        glusterd_volinfo_t *volinfo     = NULL;
-        char               *server      = NULL;
-
-        priv    = THIS->private;
+        struct stat              stbuf                  = {0,};
+        int32_t                  ret                    = -1;
+        glusterd_conf_t         *priv                   = NULL;
+        char                    *vol                    = NULL;
+        char                    *dup_volname            = NULL;
+        char                    *free_ptr               = NULL;
+        char                    *save_ptr               = NULL;
+        char                    *str_token              = NULL;
+        glusterd_volinfo_t      *volinfo                = NULL;
+        char                    *server                 = NULL;
+        const char              *volname_ptr            = NULL;
+        char                     path_prefix [PATH_MAX] = {0,};
+        xlator_t                *this                   = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (volname);
+        GF_ASSERT (path);
 
         if (strstr (volname, "gluster/")) {
                 server = strchr (volname, '/') + 1;
@@ -57,6 +178,22 @@ build_volfile_path (const char *volname, char *path,
                                               path, path_len);
                 ret = 1;
                 goto out;
+        } else if ((str_token = strstr (volname, "/snaps/"))) {
+                ret = get_snap_volname_and_volinfo (str_token, &dup_volname,
+                                                    &volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get snap"
+                                " volinfo from path (%s)", volname);
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (path_prefix, sizeof (path_prefix), "%s/snaps/%s",
+                          priv->workdir, volinfo->snapshot->snapname);
+
+                free_ptr = dup_volname;
+                volname_ptr = dup_volname;
+                goto gotvolinfo;
         } else if (volname[0] != '/') {
                 /* Normal behavior */
                 dup_volname = gf_strdup (volname);
@@ -67,39 +204,53 @@ build_volfile_path (const char *volname, char *path,
                 dup_volname = gf_strdup (&volname[1]);
         }
 
+        if (!dup_volname) {
+                gf_log(THIS->name, GF_LOG_ERROR, "strdup failed");
+                ret = -1;
+                goto out;
+        }
         free_ptr = dup_volname;
+        volname_ptr = volname;
+
+        snprintf (path_prefix, sizeof (path_prefix), "%s/vols",
+                  priv->workdir);
 
         ret = glusterd_volinfo_find (dup_volname, &volinfo);
+
         if (ret) {
                 /* Split the volume name */
-                vol = strtok_r (dup_volname, ".", &tmp);
+                vol = strtok_r (dup_volname, ".", &save_ptr);
                 if (!vol)
                         goto out;
+
                 ret = glusterd_volinfo_find (vol, &volinfo);
                 if (ret)
                         goto out;
         }
 
+gotvolinfo:
         if (!glusterd_auth_get_username (volinfo))
                 trusted_str = NULL;
 
-        ret = snprintf (path, path_len, "%s/vols/%s/%s.vol",
-                        priv->workdir, volinfo->volname, volname);
+        ret = snprintf (path, path_len, "%s/%s/%s.vol", path_prefix,
+                        volinfo->volname, volname_ptr);
         if (ret == -1)
                 goto out;
 
         ret = stat (path, &stbuf);
 
         if ((ret == -1) && (errno == ENOENT)) {
-                snprintf (path, path_len, "%s/vols/%s/%s%s-fuse.vol",
-                          priv->workdir, volinfo->volname,
-                          (trusted_str ? trusted_str : ""), dup_volname);
+                snprintf (path, path_len, "%s/%s/%s%s-fuse.vol",
+                          path_prefix, volinfo->volname,
+                          (trusted_str ? trusted_str : ""),
+                          dup_volname);
+
                 ret = stat (path, &stbuf);
         }
 
         if ((ret == -1) && (errno == ENOENT)) {
-                snprintf (path, path_len, "%s/vols/%s/%s-tcp.vol",
-                          priv->workdir, volinfo->volname, volname);
+                snprintf (path, path_len, "%s/%s/%s-tcp.vol",
+                          path_prefix, volinfo->volname, volname_ptr);
         }
 
         ret = 1;
@@ -244,16 +395,22 @@ __server_getspec (rpcsvc_request_t *req)
         }
 
         trans = req->trans;
+        /* addrstr will be empty for cli socket connections */
         ret = rpcsvc_transport_peername (trans, (char *)&addrstr,
                                          sizeof (addrstr));
         if (ret)
                 goto fail;
 
-        tmp = strrchr (addrstr, ':');
-        *tmp = '\0';
+        tmp  = strrchr (addrstr, ':');
+        if (tmp)
+                *tmp = '\0';
 
-        /* we trust the local admin */
-        if (gf_is_local_addr (addrstr)) {
+        /* The trusted volfiles are given to the glusterd owned process like NFS
+         * server, self-heal daemon etc., so that they are not inadvertently
+         * blocked by a auth.{allow,reject} setting. The trusted volfile is not
+         * meant for external users.
+         */
+        if (strlen (addrstr) && gf_is_local_addr (addrstr)) {
 
                 ret = build_volfile_path (volume, filename,
                                           sizeof (filename),
@@ -408,12 +565,16 @@ gd_validate_cluster_op_version (xlator_t *this, int cluster_op_version,
                 goto out;
         }
 
-        if (cluster_op_version < conf->op_version) {
+        /* The peer can only reduce its op-version when it doesn't have any
+         * volumes. Reducing op-version when it already contains volumes can
+         * lead to inconsistencies in the cluster
+         */
+        if ((cluster_op_version < conf->op_version) &&
+            !list_empty (&conf->volumes)) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "operating version %d is less than the currently "
-                        "running version (%d) on the machine (as per peer "
-                        "request from %s)", cluster_op_version,
-                        conf->op_version, peerid);
+                        "cannot reduce operating version to %d from current "
+                        "version %d as volumes exist (as per peer request from "
+                        "%s)", cluster_op_version, conf->op_version, peerid);
                 goto out;
         }
 
@@ -596,6 +757,19 @@ struct rpcsvc_program gluster_handshake_prog = {
         .numactors = GF_HNDSK_MAXVALUE,
 };
 
+/* A minimal RPC program just for the cli getspec command */
+rpcsvc_actor_t gluster_cli_getspec_actors[] = {
+        [GF_HNDSK_GETSPEC]      = {"GETSPEC",     GF_HNDSK_GETSPEC,      server_getspec,      NULL, 0, DRC_NA},
+};
+
+struct rpcsvc_program gluster_cli_getspec_prog = {
+        .progname  = "Gluster Handshake (CLI Getspec)",
+        .prognum   = GLUSTER_HNDSK_PROGRAM,
+        .progver   = GLUSTER_HNDSK_VERSION,
+        .actors    = gluster_cli_getspec_actors,
+        .numactors = GF_HNDSK_MAXVALUE,
+};
+
 
 char *glusterd_dump_proc[GF_DUMP_MAXVALUE] = {
         [GF_DUMP_NULL] = "NULL",
@@ -733,16 +907,6 @@ gd_validate_peer_op_version (xlator_t *this, glusterd_peerinfo_t *peerinfo,
                 goto out;
         }
 
-        /* If peer is already operating at a higher op_version reject it.
-         * Cluster cannot be moved to higher op_version to accomodate a peer.
-         */
-        if (peer_op_version > conf->op_version) {
-                ret = gf_asprintf (errstr, "Peer %s is already at a higher "
-                                   "op-version", peerinfo->hostname);
-                ret = -1;
-                goto out;
-        }
-
         ret = 0;
 out:
         gf_log (this->name , GF_LOG_DEBUG, "Peer %s %s", peerinfo->hostname,
@@ -799,6 +963,7 @@ __glusterd_mgmt_hndsk_version_ack_cbk (struct rpc_req *req, struct iovec *iov,
          */
         peerinfo->mgmt = &gd_mgmt_prog;
         peerinfo->peer = &gd_peer_prog;
+        peerinfo->mgmt_v3 = &gd_mgmt_v3_prog;
 
         ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
 
@@ -1025,6 +1190,14 @@ glusterd_set_clnt_mgmt_program (glusterd_peerinfo_t *peerinfo,
                          peerinfo->peer->progname, peerinfo->peer->prognum,
                          peerinfo->peer->progver);
         }
+
+        if (peerinfo->mgmt_v3) {
+                 gf_log ("", GF_LOG_INFO,
+                         "Using Program %s, Num (%d), Version (%d)",
+                         peerinfo->mgmt_v3->progname, peerinfo->mgmt_v3->prognum,
+                         peerinfo->mgmt_v3->progver);
+        }
+
         ret = 0;
 out:
         return ret;
diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.c b/xlators/mgmt/glusterd/src/glusterd-locks.c
new file mode 100644
index 000000000..0af2a186f
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-locks.c
@@ -0,0 +1,637 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "common-utils.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-store.h"
+#include "glusterd-utils.h"
+#include "glusterd-volgen.h"
+#include "glusterd-locks.h"
+#include "run.h"
+#include "syscall.h"
+
+#include <signal.h>
+
+#define GF_MAX_LOCKING_ENTITIES 2
+
+/* Valid entities that the mgmt_v3 lock can hold locks upon    *
+ * To add newer entities to be locked, we can just add more    *
+ * entries to this table along with the type and default value */
+valid_entities   valid_types[] = {
+        { "vol",  _gf_true  },
+        { "snap", _gf_false },
+        { NULL              },
+};
+
+static dict_t *mgmt_v3_lock;
+
+/* Checks if the lock request is for a valid entity */
+gf_boolean_t
+glusterd_mgmt_v3_is_type_valid (char *type)
+{
+        int32_t         i   = 0;
+        gf_boolean_t    ret = _gf_false;
+
+        GF_ASSERT (type);
+
+        for (i = 0; valid_types[i].type; i++) {
+                if (!strcmp (type, valid_types[i].type)) {
+                        ret = _gf_true;
+                        break;
+                }
+        }
+
+        return ret;
+}
+
+/* Initialize the global mgmt_v3 lock list(dict) when
+ * glusterd is spawned */
+int32_t
+glusterd_mgmt_v3_lock_init ()
+{
+        int32_t ret = -1;
+
+        mgmt_v3_lock = dict_new ();
+        if (!mgmt_v3_lock) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+/* Destroy the global mgmt_v3 lock list(dict) when
+ * glusterd cleanup is performed */
+void
+glusterd_mgmt_v3_lock_fini ()
+{
+        if (mgmt_v3_lock)
+                dict_destroy (mgmt_v3_lock);
+}
+
+int32_t
+glusterd_get_mgmt_v3_lock_owner (char *key, uuid_t *uuid)
+{
+        int32_t             ret      = -1;
+        mgmt_v3_lock_obj   *lock_obj = NULL;
+        uuid_t              no_owner = {"\0"};
+        xlator_t           *this = NULL;
+
+        GF_ASSERT(THIS);
+        this = THIS;
+
+        if (!key || !uuid) {
+                gf_log (this->name, GF_LOG_ERROR, "key or uuid is null.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_bin(mgmt_v3_lock, key, (void **) &lock_obj);
+        if (!ret)
+                uuid_copy (*uuid, lock_obj->lock_owner);
+        else
+                uuid_copy (*uuid, no_owner);
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* This function is called with the locked_count and type, to   *
+ * release all the acquired locks. */
+static int32_t
+glusterd_release_multiple_locks_per_entity (dict_t *dict, uuid_t uuid,
+                                            int32_t locked_count,
+                                            char *type)
+{
+        char           name_buf[PATH_MAX]    = "";
+        char          *name                  = NULL;
+        int32_t        i                     = -1;
+        int32_t        op_ret                = 0;
+        int32_t        ret                   = -1;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT (dict);
+        GF_ASSERT (type);
+
+        if (locked_count == 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "No %s locked as part of this transaction",
+                        type);
+                goto out;
+        }
+
+        /* Release all the locks held */
+        for (i = 0; i < locked_count; i++) {
+                snprintf (name_buf, sizeof(name_buf),
+                          "%sname%d", type, i+1);
+
+                /* Looking for volname1, volname2 or snapname1, *
+                 * as key in the dict snapname2 */
+                ret = dict_get_str (dict, name_buf, &name);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get %s locked_count = %d",
+                                name_buf, locked_count);
+                        op_ret = ret;
+                        continue;
+                }
+
+                ret = glusterd_mgmt_v3_unlock (name, uuid, type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to release lock for %s.",
+                                name);
+                        op_ret = ret;
+               }
+        }
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", op_ret);
+        return op_ret;
+}
+
+/* Given the count and type of the entity this function acquires     *
+ * locks on multiple elements of the same entity. For example:       *
+ * If type is "vol" this function tries to acquire locks on multiple *
+ * volumes */
+static int32_t
+glusterd_acquire_multiple_locks_per_entity (dict_t *dict, uuid_t uuid,
+                                            int32_t count, char *type)
+{
+        char           name_buf[PATH_MAX]    = "";
+        char          *name                  = NULL;
+        int32_t        i                     = -1;
+        int32_t        ret                   = -1;
+        int32_t        locked_count          = 0;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT (dict);
+        GF_ASSERT (type);
+
+        /* Locking one element after other */
+        for (i = 0; i < count; i++) {
+                snprintf (name_buf, sizeof(name_buf),
+                          "%sname%d", type, i+1);
+
+                /* Looking for volname1, volname2 or snapname1, *
+                 * as key in the dict snapname2 */
+                ret = dict_get_str (dict, name_buf, &name);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get %s count = %d",
+                                name_buf, count);
+                        break;
+                }
+
+                ret = glusterd_mgmt_v3_lock (name, uuid, type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to acquire lock for %s %s "
+                                "on behalf of %s. Reversing "
+                                "this transaction", type, name,
+                                uuid_utoa(uuid));
+                        break;
+                }
+                locked_count++;
+        }
+
+        if (count == locked_count) {
+                /* If all locking ops went successfuly, return as success */
+                ret = 0;
+                goto out;
+        }
+
+        /* If we failed to lock one element, unlock others and return failure */
+        ret = glusterd_release_multiple_locks_per_entity (dict, uuid,
+                                                          locked_count,
+                                                          type);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to release multiple %s locks",
+                        type);
+        }
+        ret = -1;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Given the type of entity, this function figures out if it should unlock a   *
+ * single element of multiple elements of the said entity. For example:        *
+ * if the type is "vol", this function will accordingly unlock a single volume *
+ * or multiple volumes */
+static int32_t
+glusterd_mgmt_v3_unlock_entity (dict_t *dict, uuid_t uuid, char *type,
+                                gf_boolean_t default_value)
+{
+        char           name_buf[PATH_MAX]    = "";
+        char          *name                  = NULL;
+        int32_t        count                 = -1;
+        int32_t        ret                   = -1;
+        gf_boolean_t   hold_locks            = _gf_false;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT (dict);
+        GF_ASSERT (type);
+
+        snprintf (name_buf, sizeof(name_buf), "hold_%s_locks", type);
+        hold_locks = dict_get_str_boolean (dict, name_buf, default_value);
+
+        if (hold_locks == _gf_false) {
+                /* Locks were not held for this particular entity *
+                 * Hence nothing to release */
+                ret = 0;
+                goto out;
+        }
+
+        /* Looking for volcount or snapcount in the dict */
+        snprintf (name_buf, sizeof(name_buf), "%scount", type);
+        ret = dict_get_int32 (dict, name_buf, &count);
+        if (ret) {
+                /* count is not present. Only one *
+                 * element name needs to be unlocked */
+                snprintf (name_buf, sizeof(name_buf), "%sname",
+                          type);
+                ret = dict_get_str (dict, name_buf, &name);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to fetch %sname", type);
+                        goto out;
+                }
+
+                ret = glusterd_mgmt_v3_unlock (name, uuid, type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to release lock for %s %s "
+                                "on behalf of %s.", type, name,
+                                uuid_utoa(uuid));
+                        goto out;
+                }
+        } else {
+                /* Unlocking one element name after another */
+                ret = glusterd_release_multiple_locks_per_entity (dict,
+                                                                  uuid,
+                                                                  count,
+                                                                  type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to release all %s locks", type);
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Given the type of entity, this function figures out if it should lock a   *
+ * single element or multiple elements of the said entity. For example:      *
+ * if the type is "vol", this function will accordingly lock a single volume *
+ * or multiple volumes */
+static int32_t
+glusterd_mgmt_v3_lock_entity (dict_t *dict, uuid_t uuid, char *type,
+                              gf_boolean_t default_value)
+{
+        char           name_buf[PATH_MAX]    = "";
+        char          *name                  = NULL;
+        int32_t        count                 = -1;
+        int32_t        ret                   = -1;
+        gf_boolean_t   hold_locks            = _gf_false;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT (dict);
+        GF_ASSERT (type);
+
+        snprintf (name_buf, sizeof(name_buf), "hold_%s_locks", type);
+        hold_locks = dict_get_str_boolean (dict, name_buf, default_value);
+
+        if (hold_locks == _gf_false) {
+                /* Not holding locks for this particular entity */
+                ret = 0;
+                goto out;
+        }
+
+        /* Looking for volcount or snapcount in the dict */
+        snprintf (name_buf, sizeof(name_buf), "%scount", type);
+        ret = dict_get_int32 (dict, name_buf, &count);
+        if (ret) {
+                /* count is not present. Only one *
+                 * element name needs to be locked */
+                snprintf (name_buf, sizeof(name_buf), "%sname",
+                          type);
+                ret = dict_get_str (dict, name_buf, &name);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to fetch %sname", type);
+                        goto out;
+                }
+
+                ret = glusterd_mgmt_v3_lock (name, uuid, type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to acquire lock for %s %s "
+                                "on behalf of %s.", type, name,
+                                uuid_utoa(uuid));
+                        goto out;
+                }
+        } else {
+                /* Locking one element name after another */
+                ret = glusterd_acquire_multiple_locks_per_entity (dict,
+                                                                  uuid,
+                                                                  count,
+                                                                  type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to acquire all %s locks", type);
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Try to release locks of multiple entities like *
+ * volume, snaps etc. */
+int32_t
+glusterd_multiple_mgmt_v3_unlock (dict_t *dict, uuid_t uuid)
+{
+        int32_t        i                     = -1;
+        int32_t        ret                   = -1;
+        int32_t        op_ret                = 0;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR, "dict is null.");
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 0; valid_types[i].type; i++) {
+                ret = glusterd_mgmt_v3_unlock_entity
+                                            (dict, uuid,
+                                             valid_types[i].type,
+                                             valid_types[i].default_value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to unlock all %s",
+                                valid_types[i].type);
+                        op_ret = ret;
+                }
+        }
+
+        ret = op_ret;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+/* Try to acquire locks on multiple entities like *
+ * volume, snaps etc. */
+int32_t
+glusterd_multiple_mgmt_v3_lock (dict_t *dict, uuid_t uuid)
+{
+        int32_t        i                     = -1;
+        int32_t        ret                   = -1;
+        int32_t        locked_count          = 0;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR, "dict is null.");
+                ret = -1;
+                goto out;
+        }
+
+        /* Locking one entity after other */
+        for (i = 0; valid_types[i].type; i++) {
+                ret = glusterd_mgmt_v3_lock_entity
+                                            (dict, uuid,
+                                             valid_types[i].type,
+                                             valid_types[i].default_value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to lock all %s",
+                                valid_types[i].type);
+                        break;
+                }
+                locked_count++;
+        }
+
+        if (locked_count == GF_MAX_LOCKING_ENTITIES) {
+                /* If all locking ops went successfuly, return as success */
+                ret = 0;
+                goto out;
+        }
+
+        /* If we failed to lock one entity, unlock others and return failure */
+        for (i = 0; i < locked_count; i++) {
+                ret = glusterd_mgmt_v3_unlock_entity
+                                              (dict, uuid,
+                                               valid_types[i].type,
+                                               valid_types[i].default_value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to unlock all %s",
+                                valid_types[i].type);
+                }
+        }
+        ret = -1;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_lock (const char *name, uuid_t uuid, char *type)
+{
+        char                 key[PATH_MAX] = "";
+        int32_t              ret           = -1;
+        mgmt_v3_lock_obj    *lock_obj      = NULL;
+        gf_boolean_t         is_valid      = _gf_true;
+        uuid_t               owner         = {0};
+        xlator_t            *this          = NULL;
+
+        GF_ASSERT(THIS);
+        this = THIS;
+
+        if (!name || !type) {
+                gf_log (this->name, GF_LOG_ERROR, "name or type is null.");
+                ret = -1;
+                goto out;
+        }
+
+        is_valid = glusterd_mgmt_v3_is_type_valid (type);
+        if (is_valid != _gf_true) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid entity. Cannot perform locking "
+                        "operation on %s types", type);
+                ret = -1;
+                goto out;
+        }
+
+        ret = snprintf (key, sizeof(key), "%s_%s", name, type);
+        if (ret != strlen(name) + 1 + strlen(type)) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create key");
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Trying to acquire lock of %s %s for %s as %s",
+                type, name, uuid_utoa (uuid), key);
+
+        ret = glusterd_get_mgmt_v3_lock_owner (key, &owner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Unable to get mgmt_v3 lock owner");
+                goto out;
+        }
+
+        /* If the lock has already been held for the given volume
+         * we fail */
+        if (!uuid_is_null (owner)) {
+                gf_log (this->name, GF_LOG_ERROR, "Lock for %s held by %s",
+                        name, uuid_utoa (owner));
+                ret = -1;
+                goto out;
+        }
+
+        lock_obj = GF_CALLOC (1, sizeof(mgmt_v3_lock_obj),
+                              gf_common_mt_mgmt_v3_lock_obj_t);
+        if (!lock_obj) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (lock_obj->lock_owner, uuid);
+
+        ret = dict_set_bin (mgmt_v3_lock, key, lock_obj,
+                            sizeof(*lock_obj));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set lock owner in mgmt_v3 lock");
+                if (lock_obj)
+                        GF_FREE (lock_obj);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Lock for %s %s successfully held by %s",
+                type, name, uuid_utoa (uuid));
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_unlock (const char *name, uuid_t uuid, char *type)
+{
+        char           key[PATH_MAX] = "";
+        int32_t        ret           = -1;
+        gf_boolean_t   is_valid      = _gf_true;
+        uuid_t         owner         = {0};
+        xlator_t            *this          = NULL;
+
+        GF_ASSERT(THIS);
+        this = THIS;
+
+        if (!name || !type) {
+                gf_log (this->name, GF_LOG_ERROR, "name is null.");
+                ret = -1;
+                goto out;
+        }
+
+        is_valid = glusterd_mgmt_v3_is_type_valid (type);
+        if (is_valid != _gf_true) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid entity. Cannot perform unlocking "
+                        "operation on %s types", type);
+                ret = -1;
+                goto out;
+        }
+
+        ret = snprintf (key, sizeof(key), "%s_%s",
+                        name, type);
+        if (ret != strlen(name) + 1 + strlen(type)) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create key");
+                ret = -1;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Trying to release lock of %s %s for %s as %s",
+                type, name, uuid_utoa (uuid), key);
+
+        ret = glusterd_get_mgmt_v3_lock_owner (key, &owner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Unable to get mgmt_v3 lock owner");
+                goto out;
+        }
+
+        if (uuid_is_null (owner)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Lock for %s %s not held", type, name);
+                ret = -1;
+                goto out;
+        }
+
+        ret = uuid_compare (uuid, owner);
+        if (ret) {
+
+                gf_log (this->name, GF_LOG_ERROR, "Lock owner mismatch. "
+                        "Lock for %s %s held by %s",
+                        type, name, uuid_utoa (owner));
+                goto out;
+        }
+
+        /* Removing the mgmt_v3 lock from the global list */
+        dict_del (mgmt_v3_lock, key);
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Lock for %s %s successfully released",
+                type, name);
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.h b/xlators/mgmt/glusterd/src/glusterd-locks.h
new file mode 100644
index 000000000..83eb8c997
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-locks.h
@@ -0,0 +1,51 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_LOCKS_H_
+#define _GLUSTERD_LOCKS_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+typedef struct mgmt_v3_lock_object_ {
+        uuid_t              lock_owner;
+} mgmt_v3_lock_obj;
+
+typedef struct mgmt_v3_lock_valid_entities {
+        char          *type;          /* Entity type like vol, snap */
+        gf_boolean_t   default_value; /* The default value that  *
+                                       * determines if the locks *
+                                       * should be held for that *
+                                       * entity */
+} valid_entities;
+
+int32_t
+glusterd_mgmt_v3_lock_init ();
+
+void
+glusterd_mgmt_v3_lock_fini ();
+
+int32_t
+glusterd_get_mgmt_v3_lock_owner (char *volname, uuid_t *uuid);
+
+int32_t
+glusterd_mgmt_v3_lock (const char *key, uuid_t uuid, char *type);
+
+int32_t
+glusterd_mgmt_v3_unlock (const char *key, uuid_t uuid, char *type);
+
+int32_t
+glusterd_multiple_mgmt_v3_lock (dict_t *dict, uuid_t uuid);
+
+int32_t
+glusterd_multiple_mgmt_v3_unlock (dict_t *dict, uuid_t uuid);
+
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-log-ops.c b/xlators/mgmt/glusterd/src/glusterd-log-ops.c
index 0136cddc9..33bd95c03 100644
--- a/xlators/mgmt/glusterd/src/glusterd-log-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-log-ops.c
@@ -24,7 +24,7 @@
 #include <signal.h>
 
 int
-glusterd_handle_log_rotate (rpcsvc_request_t *req)
+__glusterd_handle_log_rotate (rpcsvc_request_t *req)
 {
         int32_t                 ret     = -1;
         gf_cli_req              cli_req = {{0,}};
@@ -90,6 +90,13 @@ out:
         return ret;
 }
 
+int
+glusterd_handle_log_rotate (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_log_rotate);
+}
+
 /* op-sm */
 int
 glusterd_op_stage_log_rotate (dict_t *dict, char **op_errstr)
diff --git a/xlators/mgmt/glusterd/src/glusterd-mem-types.h b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
index 51057c274..e6f6a0333 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mem-types.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
@@ -67,7 +67,9 @@ typedef enum gf_gld_mem_types_ {
         gf_gld_mt_hooks_priv_t                  = gf_common_mt_end + 51,
         gf_gld_mt_mop_commit_req_t              = gf_common_mt_end + 52,
         gf_gld_mt_int                           = gf_common_mt_end + 53,
-        gf_gld_mt_end                           = gf_common_mt_end + 54,
+        gf_gld_mt_snap_t                        = gf_common_mt_end + 54,
+        gf_gld_mt_missed_snapinfo_t             = gf_common_mt_end + 55,
+        gf_gld_mt_end                           = gf_common_mt_end + 56,
 } gf_gld_mem_types_t;
 #endif
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c b/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c
new file mode 100644
index 000000000..a5f38ce9c
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c
@@ -0,0 +1,924 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+/* rpc related syncops */
+#include "rpc-clnt.h"
+#include "protocol-common.h"
+#include "xdr-generic.h"
+#include "glusterd1-xdr.h"
+#include "glusterd-syncop.h"
+
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-locks.h"
+#include "glusterd-mgmt.h"
+#include "glusterd-op-sm.h"
+
+static int
+glusterd_mgmt_v3_null (rpcsvc_request_t *req)
+{
+        return 0;
+}
+
+static int
+glusterd_mgmt_v3_lock_send_resp (rpcsvc_request_t *req, int32_t status)
+{
+
+        gd1_mgmt_v3_lock_rsp          rsp   = {{0},};
+        int                           ret   = -1;
+        xlator_t                     *this  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        if (rsp.op_ret)
+               rsp.op_errno = errno;
+
+        glusterd_get_uuid (&rsp.uuid);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Responded to mgmt_v3 lock, ret: %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_synctasked_mgmt_v3_lock (rpcsvc_request_t *req,
+                                  gd1_mgmt_v3_lock_req *lock_req,
+                                  glusterd_op_lock_ctx_t *ctx)
+{
+        int32_t                         ret         = -1;
+        xlator_t                       *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (ctx);
+        GF_ASSERT (ctx->dict);
+
+        /* Trying to acquire multiple mgmt_v3 locks */
+        ret = glusterd_multiple_mgmt_v3_lock (ctx->dict, ctx->uuid);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to acquire mgmt_v3 locks for %s",
+                         uuid_utoa (ctx->uuid));
+
+        ret = glusterd_mgmt_v3_lock_send_resp (req, ret);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_op_state_machine_mgmt_v3_lock (rpcsvc_request_t *req,
+                                        gd1_mgmt_v3_lock_req *lock_req,
+                                        glusterd_op_lock_ctx_t *ctx)
+{
+        int32_t                         ret         = -1;
+        xlator_t                       *this        = NULL;
+        glusterd_op_info_t              txn_op_info = {{0},};
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        glusterd_txn_opinfo_init (&txn_op_info, NULL, &lock_req->op,
+                                  ctx->dict, req);
+
+        ret = glusterd_set_txn_opinfo (&lock_req->txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+                goto out;
+        }
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCK,
+                                           &lock_req->txn_id, ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_LOCK");
+
+out:
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_mgmt_v3_lock_fn (rpcsvc_request_t *req)
+{
+        gd1_mgmt_v3_lock_req            lock_req    = {{0},};
+        int32_t                         ret         = -1;
+        glusterd_peerinfo_t            *peerinfo    = NULL;
+        glusterd_op_lock_ctx_t         *ctx         = NULL;
+        xlator_t                       *this        = NULL;
+        gf_boolean_t                    is_synctasked = _gf_false;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &lock_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode lock "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Received mgmt_v3 lock req "
+                "from uuid: %s", uuid_utoa (lock_req.uuid));
+
+        if (glusterd_friend_find_by_uuid (lock_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (lock_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
+        if (!ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (ctx->uuid, lock_req.uuid);
+        ctx->req = req;
+
+        ctx->dict = dict_new ();
+        if (!ctx->dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (lock_req.dict.dict_val,
+                                lock_req.dict.dict_len, &ctx->dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        is_synctasked = dict_get_str_boolean (ctx->dict,
+                                              "is_synctasked", _gf_false);
+        if (is_synctasked)
+                ret = glusterd_synctasked_mgmt_v3_lock (req, &lock_req, ctx);
+        else
+                ret = glusterd_op_state_machine_mgmt_v3_lock (req, &lock_req,
+                                                              ctx);
+
+out:
+
+        if (ret) {
+                if (ctx->dict)
+                        dict_unref (ctx->dict);
+                if (ctx)
+                        GF_FREE (ctx);
+        }
+
+        free (lock_req.dict.dict_val);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_mgmt_v3_pre_validate_send_resp (rpcsvc_request_t   *req,
+                                        int32_t op, int32_t status,
+                                        char *op_errstr, dict_t *rsp_dict)
+{
+        gd1_mgmt_v3_pre_val_rsp          rsp      = {{0},};
+        int                             ret      = -1;
+        xlator_t                       *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op = op;
+        if (op_errstr)
+                rsp.op_errstr = op_errstr;
+        else
+                rsp.op_errstr = "";
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get serialized length of dict");
+                goto out;
+        }
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+out:
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Responded to pre validation, ret: %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_pre_validate_fn (rpcsvc_request_t *req)
+{
+        int32_t                         ret       = -1;
+        gd1_mgmt_v3_pre_val_req          op_req    = {{0},};
+        glusterd_peerinfo_t            *peerinfo  = NULL;
+        xlator_t                       *this      = NULL;
+        char                           *op_errstr = NULL;
+        dict_t                         *dict      = NULL;
+        dict_t                         *rsp_dict  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to decode pre validation "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (op_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_unserialize (op_req.dict.dict_val,
+                                op_req.dict.dict_len, &dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get new dictionary");
+                return -1;
+        }
+
+        ret = gd_mgmt_v3_pre_validate_fn (op_req.op, dict, &op_errstr,
+                                         rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Pre Validation failed on operation %s",
+                        gd_op_list[op_req.op]);
+        }
+
+        ret = glusterd_mgmt_v3_pre_validate_send_resp (req, op_req.op,
+                                                      ret, op_errstr,
+                                                      rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to send Pre Validation "
+                        "response for operation %s",
+                        gd_op_list[op_req.op]);
+                goto out;
+        }
+
+out:
+        if (op_errstr && (strcmp (op_errstr, "")))
+                GF_FREE (op_errstr);
+
+        free (op_req.dict.dict_val);
+
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_mgmt_v3_brick_op_send_resp (rpcsvc_request_t   *req,
+                                    int32_t op, int32_t status,
+                                    char *op_errstr, dict_t *rsp_dict)
+{
+        gd1_mgmt_v3_brick_op_rsp         rsp      = {{0},};
+        int                             ret      = -1;
+        xlator_t                       *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op = op;
+        if (op_errstr)
+                rsp.op_errstr = op_errstr;
+        else
+                rsp.op_errstr = "";
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get serialized length of dict");
+                goto out;
+        }
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+out:
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Responded to brick op, ret: %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_brick_op_fn (rpcsvc_request_t *req)
+{
+        int32_t                         ret       = -1;
+        gd1_mgmt_v3_brick_op_req         op_req    = {{0},};
+        glusterd_peerinfo_t            *peerinfo  = NULL;
+        xlator_t                       *this      = NULL;
+        char                           *op_errstr = NULL;
+        dict_t                         *dict      = NULL;
+        dict_t                         *rsp_dict  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode brick op "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (op_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_unserialize (op_req.dict.dict_val,
+                                op_req.dict.dict_len, &dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get new dictionary");
+                return -1;
+        }
+
+        ret = gd_mgmt_v3_brick_op_fn (op_req.op, dict, &op_errstr,
+                                     rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Brick Op failed on operation %s",
+                        gd_op_list[op_req.op]);
+        }
+
+        ret = glusterd_mgmt_v3_brick_op_send_resp (req, op_req.op,
+                                                  ret, op_errstr,
+                                                  rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to send brick op "
+                        "response for operation %s",
+                        gd_op_list[op_req.op]);
+                goto out;
+        }
+
+out:
+        if (op_errstr && (strcmp (op_errstr, "")))
+                GF_FREE (op_errstr);
+
+        free (op_req.dict.dict_val);
+
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_mgmt_v3_commit_send_resp (rpcsvc_request_t   *req,
+                                  int32_t op, int32_t status,
+                                  char *op_errstr, dict_t *rsp_dict)
+{
+        gd1_mgmt_v3_commit_rsp           rsp      = {{0},};
+        int                             ret      = -1;
+        xlator_t                       *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op = op;
+        if (op_errstr)
+                rsp.op_errstr = op_errstr;
+        else
+                rsp.op_errstr = "";
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get serialized length of dict");
+                goto out;
+        }
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_commit_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Responded to commit, ret: %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_commit_fn (rpcsvc_request_t *req)
+{
+        int32_t                         ret       = -1;
+        gd1_mgmt_v3_commit_req           op_req    = {{0},};
+        glusterd_peerinfo_t            *peerinfo  = NULL;
+        xlator_t                       *this      = NULL;
+        char                           *op_errstr = NULL;
+        dict_t                         *dict      = NULL;
+        dict_t                         *rsp_dict  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_commit_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode commit "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (op_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_unserialize (op_req.dict.dict_val,
+                                op_req.dict.dict_len, &dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get new dictionary");
+                return -1;
+        }
+
+        ret = gd_mgmt_v3_commit_fn (op_req.op, dict, &op_errstr,
+                                   rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "commit failed on operation %s",
+                        gd_op_list[op_req.op]);
+        }
+
+        ret = glusterd_mgmt_v3_commit_send_resp (req, op_req.op,
+                                                ret, op_errstr,
+                                                rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to send commit "
+                        "response for operation %s",
+                        gd_op_list[op_req.op]);
+                goto out;
+        }
+
+out:
+        if (op_errstr && (strcmp (op_errstr, "")))
+                GF_FREE (op_errstr);
+
+        free (op_req.dict.dict_val);
+
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_mgmt_v3_post_validate_send_resp (rpcsvc_request_t   *req,
+                                         int32_t op, int32_t status,
+                                         char *op_errstr, dict_t *rsp_dict)
+{
+        gd1_mgmt_v3_post_val_rsp         rsp      = {{0},};
+        int                             ret      = -1;
+        xlator_t                       *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op = op;
+        if (op_errstr)
+                rsp.op_errstr = op_errstr;
+        else
+                rsp.op_errstr = "";
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get serialized length of dict");
+                goto out;
+        }
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_post_val_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+out:
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Responded to post validation, ret: %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_post_validate_fn (rpcsvc_request_t *req)
+{
+        int32_t                         ret       = -1;
+        gd1_mgmt_v3_post_val_req         op_req    = {{0},};
+        glusterd_peerinfo_t            *peerinfo  = NULL;
+        xlator_t                       *this      = NULL;
+        char                           *op_errstr = NULL;
+        dict_t                         *dict      = NULL;
+        dict_t                         *rsp_dict  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_post_val_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to decode post validation "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (op_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_unserialize (op_req.dict.dict_val,
+                                op_req.dict.dict_len, &dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get new dictionary");
+                return -1;
+        }
+
+        ret = gd_mgmt_v3_post_validate_fn (op_req.op, op_req.op_ret, dict,
+                                           &op_errstr, rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Post Validation failed on operation %s",
+                        gd_op_list[op_req.op]);
+        }
+
+        ret = glusterd_mgmt_v3_post_validate_send_resp (req, op_req.op,
+                                                       ret, op_errstr,
+                                                       rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to send Post Validation "
+                        "response for operation %s",
+                        gd_op_list[op_req.op]);
+                goto out;
+        }
+
+out:
+        if (op_errstr && (strcmp (op_errstr, "")))
+                GF_FREE (op_errstr);
+
+        free (op_req.dict.dict_val);
+
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_mgmt_v3_unlock_send_resp (rpcsvc_request_t *req, int32_t status)
+{
+
+        gd1_mgmt_v3_unlock_rsp          rsp   = {{0},};
+        int                             ret   = -1;
+        xlator_t                       *this  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        if (rsp.op_ret)
+               rsp.op_errno = errno;
+
+        glusterd_get_uuid (&rsp.uuid);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Responded to mgmt_v3 unlock, ret: %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_synctasked_mgmt_v3_unlock (rpcsvc_request_t *req,
+                                    gd1_mgmt_v3_unlock_req *unlock_req,
+                                    glusterd_op_lock_ctx_t *ctx)
+{
+        int32_t                         ret         = -1;
+        xlator_t                       *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (ctx);
+
+        /* Trying to release multiple mgmt_v3 locks */
+        ret = glusterd_multiple_mgmt_v3_unlock (ctx->dict, ctx->uuid);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to release mgmt_v3 locks for %s",
+                        uuid_utoa(ctx->uuid));
+        }
+
+        ret = glusterd_mgmt_v3_unlock_send_resp (req, ret);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+
+static int
+glusterd_op_state_machine_mgmt_v3_unlock (rpcsvc_request_t *req,
+                                          gd1_mgmt_v3_unlock_req *lock_req,
+                                          glusterd_op_lock_ctx_t *ctx)
+{
+        int32_t                           ret      = -1;
+        xlator_t                         *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_UNLOCK,
+                                           &lock_req->txn_id, ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_UNLOCK");
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_mgmt_v3_unlock_fn (rpcsvc_request_t *req)
+{
+        gd1_mgmt_v3_unlock_req            lock_req = {{0},};
+        int32_t                           ret      = -1;
+        glusterd_op_lock_ctx_t           *ctx      = NULL;
+        glusterd_peerinfo_t              *peerinfo = NULL;
+        xlator_t                         *this     = NULL;
+        gf_boolean_t                      is_synctasked = _gf_false;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &lock_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode unlock "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Received mgmt_v3 unlock req "
+                "from uuid: %s", uuid_utoa (lock_req.uuid));
+
+        if (glusterd_friend_find_by_uuid (lock_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (lock_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
+        if (!ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (ctx->uuid, lock_req.uuid);
+        ctx->req = req;
+
+        ctx->dict = dict_new ();
+        if (!ctx->dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (lock_req.dict.dict_val,
+                                lock_req.dict.dict_len, &ctx->dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        is_synctasked = dict_get_str_boolean (ctx->dict,
+                                              "is_synctasked", _gf_false);
+        if (is_synctasked)
+                ret = glusterd_synctasked_mgmt_v3_unlock (req, &lock_req, ctx);
+        else
+                ret = glusterd_op_state_machine_mgmt_v3_unlock (req, &lock_req,
+                                                                ctx);
+
+out:
+
+        if (ret) {
+                if (ctx->dict)
+                        dict_unref (ctx->dict);
+                if (ctx)
+                        GF_FREE (ctx);
+        }
+
+        free (lock_req.dict.dict_val);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_handle_mgmt_v3_lock (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_mgmt_v3_lock_fn);
+}
+
+static int
+glusterd_handle_pre_validate (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_pre_validate_fn);
+}
+
+static int
+glusterd_handle_brick_op (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_brick_op_fn);
+}
+
+static int
+glusterd_handle_commit (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_commit_fn);
+}
+
+static int
+glusterd_handle_post_validate (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_post_validate_fn);
+}
+
+int
+glusterd_handle_mgmt_v3_unlock (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_mgmt_v3_unlock_fn);
+}
+
+rpcsvc_actor_t gd_svc_mgmt_v3_actors[] = {
+        [GLUSTERD_MGMT_V3_NULL]          = { "NULL",           GLUSTERD_MGMT_V3_NULL,          glusterd_mgmt_v3_null,          NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_LOCK]          = { "MGMT_V3_LOCK",   GLUSTERD_MGMT_V3_LOCK,          glusterd_handle_mgmt_v3_lock,   NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_PRE_VALIDATE]  = { "PRE_VAL",        GLUSTERD_MGMT_V3_PRE_VALIDATE,  glusterd_handle_pre_validate,   NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_BRICK_OP]      = { "BRCK_OP",        GLUSTERD_MGMT_V3_BRICK_OP,      glusterd_handle_brick_op,       NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_COMMIT]        = { "COMMIT",         GLUSTERD_MGMT_V3_COMMIT,        glusterd_handle_commit,         NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_POST_VALIDATE] = { "POST_VAL",       GLUSTERD_MGMT_V3_POST_VALIDATE, glusterd_handle_post_validate,  NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_UNLOCK]        = { "MGMT_V3_UNLOCK", GLUSTERD_MGMT_V3_UNLOCK,        glusterd_handle_mgmt_v3_unlock, NULL, 0, DRC_NA},
+};
+
+struct rpcsvc_program gd_svc_mgmt_v3_prog = {
+        .progname  = "GlusterD svc mgmt v3",
+        .prognum   = GD_MGMT_V3_PROGRAM,
+        .progver   = GD_MGMT_V3_VERSION,
+        .numactors = GLUSTERD_MGMT_V3_MAXVALUE,
+        .actors    = gd_svc_mgmt_v3_actors,
+        .synctask  = _gf_true,
+};
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
new file mode 100644
index 000000000..d52532e54
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
@@ -0,0 +1,1893 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+/* rpc related syncops */
+#include "rpc-clnt.h"
+#include "protocol-common.h"
+#include "xdr-generic.h"
+#include "glusterd1-xdr.h"
+#include "glusterd-syncop.h"
+
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-locks.h"
+#include "glusterd-mgmt.h"
+#include "glusterd-op-sm.h"
+
+extern struct rpc_clnt_program gd_mgmt_v3_prog;
+
+
+static void
+gd_mgmt_v3_collate_errors (struct syncargs *args, int op_ret, int op_errno,
+                           char *op_errstr, int op_code,
+                           glusterd_peerinfo_t *peerinfo, u_char *uuid)
+{
+        char      *peer_str          = NULL;
+        char       err_str[PATH_MAX] = "Please check log file for details.";
+        char       op_err[PATH_MAX]  = "";
+        int32_t    len               = -1;
+        xlator_t  *this              = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (args);
+        GF_ASSERT (uuid);
+
+        if (op_ret) {
+                args->op_ret = op_ret;
+                args->op_errno = op_errno;
+
+                if (peerinfo)
+                        peer_str = peerinfo->hostname;
+                else
+                        peer_str = uuid_utoa (uuid);
+
+                if (op_errstr && strcmp (op_errstr, "")) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "Error: %s", op_errstr);
+                        err_str[len] = '\0';
+                }
+
+                switch (op_code){
+                case GLUSTERD_MGMT_V3_LOCK:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Locking failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                case GLUSTERD_MGMT_V3_PRE_VALIDATE:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Pre Validation failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                case GLUSTERD_MGMT_V3_BRICK_OP:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Brick ops failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                case GLUSTERD_MGMT_V3_COMMIT:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Commit failed on %s. %s",
+                                                peer_str, err_str);
+                                break;
+                        }
+                case GLUSTERD_MGMT_V3_POST_VALIDATE:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Post Validation failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                case GLUSTERD_MGMT_V3_UNLOCK:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Unlocking failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                }
+                op_err[len] = '\0';
+
+                if (args->errstr) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s\n%s", args->errstr,
+                                        op_err);
+                        GF_FREE (args->errstr);
+                        args->errstr = NULL;
+                } else
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s", op_err);
+                err_str[len] = '\0';
+
+                gf_log (this->name, GF_LOG_ERROR, "%s", op_err);
+                args->errstr = gf_strdup (err_str);
+        }
+
+        return;
+}
+
+int32_t
+gd_mgmt_v3_pre_validate_fn (glusterd_op_t op, dict_t *dict,
+                            char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t       ret = -1;
+        xlator_t     *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        switch (op) {
+               case GD_OP_SNAP:
+               {
+                       ret = glusterd_snapshot_prevalidate (dict, op_errstr,
+                                                            rsp_dict);
+                       if (ret) {
+                               gf_log (this->name, GF_LOG_WARNING,
+                                       "Snapshot Prevalidate Failed");
+                               goto out;
+                       }
+
+                       break;
+               }
+               default:
+                       break;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "OP = %d. Returning %d", op, ret);
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_brick_op_fn (glusterd_op_t op, dict_t *dict,
+                        char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t       ret = -1;
+        xlator_t     *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        switch (op) {
+        case GD_OP_SNAP:
+        {
+                ret = glusterd_snapshot_brickop (dict, op_errstr, rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "snapshot brickop "
+                                "failed");
+                        goto out;
+                }
+                break;
+        }
+        default:
+                break;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "OP = %d. Returning %d", op, ret);
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_commit_fn (glusterd_op_t op, dict_t *dict,
+                      char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t       ret = -1;
+        xlator_t     *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        switch (op) {
+               case GD_OP_SNAP:
+               {
+                       ret = glusterd_snapshot (dict, op_errstr, rsp_dict);
+                       if (ret) {
+                               gf_log (this->name, GF_LOG_WARNING,
+                                       "Snapshot Commit Failed");
+                               goto out;
+                       }
+                       break;
+               }
+               default:
+                       break;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "OP = %d. Returning %d", op, ret);
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_post_validate_fn (glusterd_op_t op, int32_t op_ret, dict_t *dict,
+                             char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t       ret = -1;
+        xlator_t     *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        switch (op) {
+               case GD_OP_SNAP:
+               {
+                       ret = glusterd_snapshot_postvalidate (dict, op_ret,
+                                                             op_errstr,
+                                                             rsp_dict);
+                       if (ret) {
+                               gf_log (this->name, GF_LOG_WARNING,
+                                       "postvalidate operation failed");
+                               goto out;
+                       }
+                       break;
+               }
+               default:
+                       break;
+        }
+
+        ret = 0;
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "OP = %d. Returning %d", op, ret);
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_lock_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_lock_rsp        rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        /* Even though the lock command has failed, while collating the errors
+           (gd_mgmt_v3_collate_errors), args->op_ret and args->op_errno will be
+           used. @args is obtained from frame->local. So before checking the
+           status of the request and going out if its a failure, args should be
+           set to frame->local. Otherwise, while collating args will be NULL.
+           This applies to other phases such as prevalidate, brickop, commit and
+           postvalidate also.
+        */
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_LOCK,
+                                   peerinfo, rsp.uuid);
+        free (rsp.dict.dict_val);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                     int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_lock_cbk_fn);
+}
+
+int
+gd_mgmt_v3_lock (glusterd_op_t op, dict_t *op_ctx,
+                 glusterd_peerinfo_t *peerinfo,
+                 struct syncargs *args, uuid_t my_uuid,
+                 uuid_t recv_uuid)
+{
+        gd1_mgmt_v3_lock_req     req  = {{0},};
+        glusterd_conf_t         *conf = THIS->private;
+        int32_t                  ret  = -1;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_LOCK,
+                                        gd_mgmt_v3_lock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_lock_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_initiate_lockdown (glusterd_conf_t  *conf, glusterd_op_t op,
+                                    dict_t *dict, char **op_errstr, int npeers,
+                                    gf_boolean_t  *is_acquired)
+{
+        char                *volname    = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (is_acquired);
+
+        peers = &conf->xaction_peers;
+
+        /* Trying to acquire multiple mgmt_v3 locks on local node */
+        ret = glusterd_multiple_mgmt_v3_lock (dict, MY_UUID);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to acquire mgmt_v3 locks on localhost");
+                goto out;
+        }
+
+        *is_acquired = _gf_true;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending mgmt_v3 lock req to other nodes in the cluster */
+        gd_syncargs_init (&args, NULL);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_lock (op, dict, peerinfo, &args,
+                                 MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.errstr)
+                *op_errstr = gf_strdup (args.errstr);
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent lock op req for %s "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        if (ret) {
+                if (*op_errstr)
+                        gf_log (this->name, GF_LOG_ERROR, "%s",
+                                *op_errstr);
+
+                if (volname)
+                        ret = gf_asprintf (op_errstr,
+                                           "Another transaction is in progress "
+                                           "for %s. Please try again after "
+                                           "sometime.", volname);
+                else
+                        ret = gf_asprintf (op_errstr,
+                                           "Another transaction is in progress "
+                                           "Please try again after sometime.");
+
+                if (ret == -1)
+                        *op_errstr = NULL;
+
+                ret = -1;
+        }
+
+        return ret;
+}
+
+int
+glusterd_pre_validate_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp)
+{
+        int32_t              ret      = 0;
+        xlator_t            *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (aggr);
+        GF_ASSERT (rsp);
+
+        switch (op) {
+        case GD_OP_SNAP:
+                ret = glusterd_snap_pre_validate_use_rsp_dict (aggr, rsp);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to aggregate prevalidate "
+                                "response dictionaries.");
+                        goto out;
+                }
+                break;
+        default:
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Invalid op (%s)", gd_op_list[op]);
+
+                break;
+        }
+out:
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_pre_validate_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_pre_val_rsp      rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        dict_t                     *rsp_dict   = NULL;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_rsp);
+        if (ret < 0)
+                goto out;
+
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                rsp_dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                        rsp.dict.dict_len,
+                                        &rsp_dict);
+                if (ret < 0) {
+                        GF_FREE (rsp.dict.dict_val);
+                        goto out;
+                } else {
+                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
+                }
+        }
+
+        uuid_copy (args->uuid, rsp.uuid);
+        pthread_mutex_lock (&args->lock_dict);
+        {
+                ret = glusterd_pre_validate_aggr_rsp_dict (rsp.op, args->dict,
+                                                           rsp_dict);
+        }
+        pthread_mutex_unlock (&args->lock_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        "Failed to aggregate response from "
+                        " node/brick");
+                if (!rsp.op_ret)
+                        op_ret = ret;
+                else {
+                        op_ret = rsp.op_ret;
+                        op_errno = rsp.op_errno;
+                }
+        } else {
+                op_ret = rsp.op_ret;
+                op_errno = rsp.op_errno;
+        }
+
+out:
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                  GLUSTERD_MGMT_V3_PRE_VALIDATE,
+                                  peerinfo, rsp.uuid);
+
+        if (rsp.op_errstr)
+                free (rsp.op_errstr);
+
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_pre_validate_cbk (struct rpc_req *req, struct iovec *iov,
+                             int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_pre_validate_cbk_fn);
+}
+
+int
+gd_mgmt_v3_pre_validate_req (glusterd_op_t op, dict_t *op_ctx,
+                             glusterd_peerinfo_t *peerinfo,
+                             struct syncargs *args, uuid_t my_uuid,
+                             uuid_t recv_uuid)
+{
+        int32_t                  ret   = -1;
+        gd1_mgmt_v3_pre_val_req  req   = {{0},};
+        glusterd_conf_t         *conf  = THIS->private;
+        xlator_t                *this  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_PRE_VALIDATE,
+                                        gd_mgmt_v3_pre_validate_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_pre_val_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_pre_validate (glusterd_conf_t  *conf, glusterd_op_t op,
+                               dict_t *req_dict, char **op_errstr, int npeers)
+{
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        dict_t              *rsp_dict   = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (req_dict);
+        GF_ASSERT (op_errstr);
+
+        peers = &conf->xaction_peers;
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create response dictionary");
+                goto out;
+        }
+
+        /* Pre Validation on local node */
+        ret = gd_mgmt_v3_pre_validate_fn (op, req_dict, op_errstr,
+                                          rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Pre Validation failed for "
+                        "operation %s on local node",
+                        gd_op_list[op]);
+
+                if (*op_errstr == NULL) {
+                        ret = gf_asprintf (op_errstr,
+                                           "Pre-validation failed "
+                                           "on localhost. Please "
+                                           "check log file for details");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        ret = -1;
+                }
+                goto out;
+        }
+
+        ret = glusterd_pre_validate_aggr_rsp_dict (op, req_dict,
+                                                   rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        "Failed to aggregate response from "
+                        " node/brick");
+                goto out;
+        }
+
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending Pre Validation req to other nodes in the cluster */
+        gd_syncargs_init (&args, req_dict);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_pre_validate_req (op, req_dict, peerinfo, &args,
+                                             MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Pre Validation failed on peers");
+
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent pre valaidation req for %s "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_build_payload (dict_t **req, char **op_errstr, dict_t *dict,
+                                glusterd_op_t op)
+{
+        int32_t                 ret      = -1;
+        dict_t                 *req_dict = NULL;
+        xlator_t               *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (dict);
+
+        req_dict = dict_new ();
+        if (!req_dict)
+                goto out;
+
+        switch (op) {
+                case GD_OP_SNAP:
+                        dict_copy (dict, req_dict);
+                        break;
+                default:
+                        break;
+        }
+
+        *req = req_dict;
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_brick_op_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                            int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_brick_op_rsp     rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        /* If the operation failed, then iov can be NULL. So better check the
+           status of the operation and then worry about iov (if the status of
+           the command is success)
+        */
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_BRICK_OP,
+                                   peerinfo, rsp.uuid);
+
+        if (rsp.op_errstr)
+                free (rsp.op_errstr);
+
+        free (rsp.dict.dict_val);
+
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_brick_op_cbk_fn);
+}
+
+int
+gd_mgmt_v3_brick_op_req (glusterd_op_t op, dict_t *op_ctx,
+                         glusterd_peerinfo_t *peerinfo,
+                         struct syncargs *args, uuid_t my_uuid,
+                         uuid_t recv_uuid)
+{
+        int32_t                   ret  = -1;
+        gd1_mgmt_v3_brick_op_req  req  = {{0},};
+        glusterd_conf_t          *conf = THIS->private;
+        xlator_t                 *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_BRICK_OP,
+                                        gd_mgmt_v3_brick_op_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_brick_op_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_brick_op (glusterd_conf_t  *conf, glusterd_op_t op,
+                           dict_t *req_dict, char **op_errstr, int npeers)
+{
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        dict_t              *rsp_dict   = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (req_dict);
+        GF_ASSERT (op_errstr);
+
+        peers = &conf->xaction_peers;
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create response dictionary");
+                goto out;
+        }
+
+        /* Perform brick op on local node */
+        ret = gd_mgmt_v3_brick_op_fn (op, req_dict, op_errstr,
+                                     rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Brick ops failed for "
+                        "operation %s on local node",
+                        gd_op_list[op]);
+
+                if (*op_errstr == NULL) {
+                        ret = gf_asprintf (op_errstr,
+                                           "Brick ops failed "
+                                           "on localhost. Please "
+                                           "check log file for details");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        ret = -1;
+                }
+                goto out;
+        }
+
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending brick op req to other nodes in the cluster */
+        gd_syncargs_init (&args, NULL);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_brick_op_req (op, req_dict, peerinfo, &args,
+                                         MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Brick ops failed on peers");
+
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent brick op req for %s "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_commit_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_commit_rsp       rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        dict_t                     *rsp_dict   = NULL;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_commit_rsp);
+        if (ret < 0)
+                goto out;
+
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                rsp_dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                        rsp.dict.dict_len,
+                                        &rsp_dict);
+                if (ret < 0) {
+                        GF_FREE (rsp.dict.dict_val);
+                        goto out;
+                } else {
+                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
+                }
+        }
+
+        uuid_copy (args->uuid, rsp.uuid);
+        pthread_mutex_lock (&args->lock_dict);
+        {
+                ret = glusterd_syncop_aggr_rsp_dict (rsp.op, args->dict,
+                                                     rsp_dict);
+        }
+        pthread_mutex_unlock (&args->lock_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        "Failed to aggregate response from "
+                        " node/brick");
+                if (!rsp.op_ret)
+                        op_ret = ret;
+                else {
+                        op_ret = rsp.op_ret;
+                        op_errno = rsp.op_errno;
+                }
+        } else {
+                op_ret = rsp.op_ret;
+                op_errno = rsp.op_errno;
+        }
+
+out:
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                  GLUSTERD_MGMT_V3_COMMIT,
+                                  peerinfo, rsp.uuid);
+
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_commit_cbk (struct rpc_req *req, struct iovec *iov,
+                       int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_commit_cbk_fn);
+}
+
+int
+gd_mgmt_v3_commit_req (glusterd_op_t op, dict_t *op_ctx,
+                       glusterd_peerinfo_t *peerinfo,
+                       struct syncargs *args, uuid_t my_uuid,
+                       uuid_t recv_uuid)
+{
+        int32_t                  ret  = -1;
+        gd1_mgmt_v3_commit_req   req  = {{0},};
+        glusterd_conf_t         *conf = THIS->private;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_COMMIT,
+                                        gd_mgmt_v3_commit_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_commit_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_commit (glusterd_conf_t  *conf, glusterd_op_t op,
+                         dict_t *op_ctx, dict_t *req_dict,
+                         char **op_errstr, int npeers)
+{
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        dict_t              *rsp_dict   = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (req_dict);
+        GF_ASSERT (op_errstr);
+
+        peers = &conf->xaction_peers;
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create response dictionary");
+                goto out;
+        }
+
+        /* Commit on local node */
+        ret = gd_mgmt_v3_commit_fn (op, req_dict, op_errstr,
+                                   rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Commit failed for "
+                        "operation %s on local node",
+                        gd_op_list[op]);
+
+                if (*op_errstr == NULL) {
+                        ret = gf_asprintf (op_errstr,
+                                           "Commit failed "
+                                           "on localhost. Please "
+                                           "check log file for details.");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        ret = -1;
+                }
+                goto out;
+        }
+
+        ret = glusterd_syncop_aggr_rsp_dict (op, op_ctx,
+                                             rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        "Failed to aggregate response from "
+                        " node/brick");
+                goto out;
+        }
+
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending commit req to other nodes in the cluster */
+        gd_syncargs_init (&args, op_ctx);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_commit_req (op, req_dict, peerinfo, &args,
+                                       MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Commit failed on peers");
+
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent commit req for %s to %d "
+                "peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_post_validate_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_post_val_rsp     rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_post_val_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                  GLUSTERD_MGMT_V3_POST_VALIDATE,
+                                  peerinfo, rsp.uuid);
+        if (rsp.op_errstr)
+                free (rsp.op_errstr);
+
+        free (rsp.dict.dict_val);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_post_validate_cbk (struct rpc_req *req, struct iovec *iov,
+                              int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_post_validate_cbk_fn);
+}
+
+int
+gd_mgmt_v3_post_validate_req (glusterd_op_t op, int32_t op_ret, dict_t *op_ctx,
+                              glusterd_peerinfo_t *peerinfo,
+                              struct syncargs *args, uuid_t my_uuid,
+                              uuid_t recv_uuid)
+{
+        int32_t                   ret  = -1;
+        gd1_mgmt_v3_post_val_req  req  = {{0},};
+        glusterd_conf_t          *conf = THIS->private;
+        xlator_t                 *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        req.op_ret = op_ret;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_POST_VALIDATE,
+                                        gd_mgmt_v3_post_validate_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_post_val_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_post_validate (glusterd_conf_t  *conf, glusterd_op_t op,
+                                int32_t op_ret, dict_t *dict, dict_t *req_dict,
+                                char **op_errstr, int npeers)
+{
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        dict_t              *rsp_dict   = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (dict);
+        GF_ASSERT (req_dict);
+        GF_ASSERT (op_errstr);
+
+        peers = &conf->xaction_peers;
+        GF_ASSERT (peers);
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create response dictionary");
+                goto out;
+        }
+
+        /* Copy the contents of dict like missed snaps info to req_dict */
+        dict_copy (dict, req_dict);
+
+        /* Post Validation on local node */
+        ret = gd_mgmt_v3_post_validate_fn (op, op_ret, req_dict, op_errstr,
+                                           rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Post Validation failed for "
+                        "operation %s on local node",
+                        gd_op_list[op]);
+
+                if (*op_errstr == NULL) {
+                        ret = gf_asprintf (op_errstr,
+                                           "Post-validation failed "
+                                           "on localhost. Please check "
+                                           "log file for details");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        ret = -1;
+                }
+                goto out;
+        }
+
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending Post Validation req to other nodes in the cluster */
+        gd_syncargs_init (&args, req_dict);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_post_validate_req (op, op_ret, req_dict, peerinfo,
+                                              &args, MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Post Validation failed on peers");
+
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent post valaidation req for %s "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_unlock_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_unlock_rsp      rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_UNLOCK,
+                                   peerinfo, rsp.uuid);
+        free (rsp.dict.dict_val);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                       int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_unlock_cbk_fn);
+}
+
+int
+gd_mgmt_v3_unlock (glusterd_op_t op, dict_t *op_ctx,
+                   glusterd_peerinfo_t *peerinfo,
+                   struct syncargs *args, uuid_t my_uuid,
+                   uuid_t recv_uuid)
+{
+        int32_t                  ret  = -1;
+        gd1_mgmt_v3_unlock_req   req  = {{0},};
+        glusterd_conf_t         *conf = THIS->private;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_UNLOCK,
+                                        gd_mgmt_v3_unlock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_unlock_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_release_peer_locks (glusterd_conf_t  *conf, glusterd_op_t op,
+                                     dict_t *dict, int32_t op_ret,
+                                     char **op_errstr, int npeers,
+                                     gf_boolean_t  is_acquired)
+{
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        peers = &conf->xaction_peers;
+
+        /* If the lock has not been held during this
+         * transaction, do not send unlock requests */
+        if (!is_acquired)
+                goto out;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending mgmt_v3 unlock req to other nodes in the cluster */
+        gd_syncargs_init (&args, NULL);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_unlock (op, dict, peerinfo, &args,
+                                   MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unlock failed on peers");
+
+                if (!op_ret && args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent unlock op req for %s "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op,
+                                      dict_t *dict)
+{
+        int32_t                     ret              = -1;
+        int32_t                     op_ret           = -1;
+        int32_t                     npeers           = 0;
+        dict_t                      *req_dict        = NULL;
+        dict_t                      *tmp_dict        = NULL;
+        glusterd_conf_t             *conf            = NULL;
+        char                        *op_errstr       = NULL;
+        xlator_t                    *this            = NULL;
+        gf_boolean_t                is_acquired      = _gf_false;
+        uuid_t                      *originator_uuid = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        /* Save the MY_UUID as the originator_uuid. This originator_uuid
+         * will be used by is_origin_glusterd() to determine if a node
+         * is the originator node for a command. */
+        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
+                                     gf_common_mt_uuid_t);
+        if (!originator_uuid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*originator_uuid, MY_UUID);
+        ret = dict_set_bin (dict, "originator_uuid",
+                            originator_uuid, sizeof (uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set originator_uuid.");
+                goto out;
+        }
+
+        /* Marking the operation as complete synctasked */
+        ret = dict_set_int32 (dict, "is_synctasked", _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set synctasked flag.");
+                goto out;
+        }
+
+        /* Use a copy at local unlock as cli response will be sent before
+         * the unlock and the volname in the dict might be removed */
+        tmp_dict = dict_new();
+        if (!tmp_dict) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create dict");
+                goto out;
+        }
+        dict_copy (dict, tmp_dict);
+
+        /* BUILD PEERS LIST */
+        INIT_LIST_HEAD (&conf->xaction_peers);
+        npeers = gd_build_peers_list  (&conf->peers, &conf->xaction_peers, op);
+
+        /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
+        ret = glusterd_mgmt_v3_initiate_lockdown (conf, op, dict, &op_errstr,
+                                                  npeers, &is_acquired);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "mgmt_v3 lockdown failed.");
+                goto out;
+        }
+
+        /* BUILD PAYLOAD */
+        ret = glusterd_mgmt_v3_build_payload (&req_dict, &op_errstr, dict, op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+                goto out;
+        }
+
+        /* PRE-COMMIT VALIDATE PHASE */
+        ret = glusterd_mgmt_v3_pre_validate (conf, op, req_dict,
+                                             &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Pre Validation Failed");
+                goto out;
+        }
+
+        /* COMMIT OP PHASE */
+        ret = glusterd_mgmt_v3_commit (conf, op, dict, req_dict,
+                                       &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Commit Op Failed");
+                goto out;
+        }
+
+        /* POST-COMMIT VALIDATE PHASE */
+        /* As of now, post_validate is not handling any other
+           commands other than snapshot. So as of now, I am
+           sending 0 (op_ret as 0).
+        */
+        ret = glusterd_mgmt_v3_post_validate (conf, op, 0, dict, req_dict,
+                                              &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Post Validation Failed");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        op_ret = ret;
+        /* UNLOCK PHASE FOR PEERS*/
+        (void) glusterd_mgmt_v3_release_peer_locks (conf, op, dict,
+                                                    op_ret, &op_errstr,
+                                                    npeers, is_acquired);
+
+        /* LOCAL VOLUME(S) UNLOCK */
+        if (is_acquired) {
+                /* Trying to release multiple mgmt_v3 locks */
+                ret = glusterd_multiple_mgmt_v3_unlock (tmp_dict, MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to release mgmt_v3 locks on localhost");
+                        op_ret = ret;
+                }
+        }
+
+        /* SEND CLI RESPONSE */
+        glusterd_op_send_cli_response (op, op_ret, 0, req, dict, op_errstr);
+
+        if (req_dict)
+                dict_unref (req_dict);
+
+        if (tmp_dict)
+                dict_unref (tmp_dict);
+
+        if (op_errstr) {
+                GF_FREE (op_errstr);
+                op_errstr = NULL;
+        }
+
+        return 0;
+}
+
+int32_t
+glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op,
+                                       dict_t *dict)
+{
+        int32_t                     ret              = -1;
+        int32_t                     op_ret           = -1;
+        int32_t                     npeers           = 0;
+        dict_t                      *req_dict        = NULL;
+        dict_t                      *tmp_dict        = NULL;
+        glusterd_conf_t             *conf            = NULL;
+        char                        *op_errstr       = NULL;
+        xlator_t                    *this            = NULL;
+        gf_boolean_t                is_acquired      = _gf_false;
+        uuid_t                      *originator_uuid = NULL;
+        gf_boolean_t                success          = _gf_false;
+        char                        *tmp_errstr      = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        /* Save the MY_UUID as the originator_uuid. This originator_uuid
+         * will be used by is_origin_glusterd() to determine if a node
+         * is the originator node for a command. */
+        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
+                                     gf_common_mt_uuid_t);
+        if (!originator_uuid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*originator_uuid, MY_UUID);
+        ret = dict_set_bin (dict, "originator_uuid",
+                            originator_uuid, sizeof (uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set originator_uuid.");
+                goto out;
+        }
+
+        /* Marking the operation as complete synctasked */
+        ret = dict_set_int32 (dict, "is_synctasked", _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set synctasked flag.");
+                goto out;
+        }
+
+        /* Use a copy at local unlock as cli response will be sent before
+         * the unlock and the volname in the dict might be removed */
+        tmp_dict = dict_new();
+        if (!tmp_dict) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create dict");
+                goto out;
+        }
+        dict_copy (dict, tmp_dict);
+
+        /* BUILD PEERS LIST */
+        INIT_LIST_HEAD (&conf->xaction_peers);
+        npeers = gd_build_peers_list  (&conf->peers, &conf->xaction_peers, op);
+
+        /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
+        ret = glusterd_mgmt_v3_initiate_lockdown (conf, op, dict, &op_errstr,
+                                                  npeers, &is_acquired);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "mgmt_v3 lockdown failed.");
+                goto out;
+        }
+
+        /* BUILD PAYLOAD */
+        ret = glusterd_mgmt_v3_build_payload (&req_dict, &op_errstr, dict, op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+                goto out;
+        }
+
+        /* PRE-COMMIT VALIDATE PHASE */
+        ret = glusterd_mgmt_v3_pre_validate (conf, op, req_dict,
+                                            &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Pre Validation Failed");
+                goto out;
+        }
+
+        /* BRICK OP PHASE for initiating barrier*/
+        ret = dict_set_int32 (req_dict, "barrier", 1);
+        if (ret)
+                goto out;
+        ret = glusterd_mgmt_v3_brick_op (conf, op, req_dict,
+                                        &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Brick Ops Failed");
+                goto unbarrier;
+        }
+
+        /* COMMIT OP PHASE */
+        /* TODO: As of now, the plan is to do quorum check before sending the
+           commit fop and if the quorum succeeds, then commit is sent to all
+           the other glusterds.
+           snap create functionality now creates the in memory and on disk
+           objects for the snapshot (marking them as incomplete), takes the lvm
+           snapshot and then updates the status of the in memory and on disk
+           snap objects as complete. Suppose one of the glusterds goes down
+           after taking the lvm snapshot, but before updating the snap object,
+           then treat it as a snapshot create failure and trigger cleanup.
+           i.e the number of commit responses received by the originator
+           glusterd shold be the same as the number of peers it has sent the
+           request to (i.e npeers variable). If not, then originator glusterd
+           will initiate cleanup in post-validate fop.
+           Question: What if one of the other glusterds goes down as explained
+           above and along with it the originator glusterd also goes down?
+           Who will initiate the cleanup?
+        */
+        ret = glusterd_mgmt_v3_commit (conf, op, dict, req_dict,
+                                       &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Commit Op Failed");
+                /* If the main op fails, we should save the error string.
+                   Because, op_errstr will be used for unbarrier and
+                   unlock ops also. We might lose the actual error that
+                   caused the failure.
+                */
+                tmp_errstr = op_errstr;
+                op_errstr = NULL;
+                goto unbarrier;
+        }
+
+        success = _gf_true;
+unbarrier:
+        /* BRICK OP PHASE for removing the barrier*/
+        ret = dict_set_int32 (req_dict, "barrier", 0);
+        if (ret)
+                goto out;
+        ret = glusterd_mgmt_v3_brick_op (conf, op, req_dict,
+                                         &op_errstr, npeers);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Brick Ops Failed");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        op_ret = ret;
+
+        if (success == _gf_false)
+                op_ret = -1;
+
+        /* POST-COMMIT VALIDATE PHASE */
+        ret = glusterd_mgmt_v3_post_validate (conf, op, op_ret, dict, req_dict,
+                                              &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Post Validation Failed");
+                op_ret = -1;
+        }
+
+        /* UNLOCK PHASE FOR PEERS*/
+        (void) glusterd_mgmt_v3_release_peer_locks (conf, op, dict,
+                                                    op_ret, &op_errstr,
+                                                    npeers, is_acquired);
+
+        /* If the commit op (snapshot taking) failed, then the error is stored
+           in tmp_errstr and unbarrier is called. Suppose, if unbarrier also
+           fails, then the error happened in unbarrier is logged and freed.
+           The error happened in commit op, which is stored in tmp_errstr
+           is sent to cli.
+        */
+        if (tmp_errstr) {
+                if (op_errstr) {
+                        gf_log (this->name, GF_LOG_ERROR, "unbarrier brick op"
+                                "failed with the error %s", op_errstr);
+                        GF_FREE (op_errstr);
+                        op_errstr = NULL;
+                }
+                op_errstr = tmp_errstr;
+        }
+
+        /* LOCAL VOLUME(S) UNLOCK */
+        if (is_acquired) {
+                /* Trying to release multiple mgmt_v3 locks */
+                ret = glusterd_multiple_mgmt_v3_unlock (tmp_dict, MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to release mgmt_v3 locks on localhost");
+                        op_ret = ret;
+                }
+        }
+
+        /* SEND CLI RESPONSE */
+        glusterd_op_send_cli_response (op, op_ret, 0, req, dict, op_errstr);
+
+        if (req_dict)
+                dict_unref (req_dict);
+
+        if (tmp_dict)
+                dict_unref (tmp_dict);
+
+        if (op_errstr) {
+                GF_FREE (op_errstr);
+                op_errstr = NULL;
+        }
+
+        return 0;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-mgmt.h
new file mode 100644
index 000000000..b185a9bec
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.h
@@ -0,0 +1,45 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_MGMT_H_
+#define _GLUSTERD_MGMT_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+int32_t
+gd_mgmt_v3_pre_validate_fn (glusterd_op_t op, dict_t *dict,
+                           char **op_errstr, dict_t *rsp_dict);
+
+int32_t
+gd_mgmt_v3_brick_op_fn (glusterd_op_t op, dict_t *dict,
+                       char **op_errstr, dict_t *rsp_dict);
+
+int32_t
+gd_mgmt_v3_commit_fn (glusterd_op_t op, dict_t *dict,
+                     char **op_errstr, dict_t *rsp_dict);
+
+int32_t
+gd_mgmt_v3_post_validate_fn (glusterd_op_t op, int32_t op_ret, dict_t *dict,
+                            char **op_errstr, dict_t *rsp_dict);
+
+int32_t
+glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op,
+                                     dict_t *dict);
+
+int32_t
+glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op,
+                                      dict_t *dict);
+
+int
+glusterd_snap_pre_validate_use_rsp_dict (dict_t *dst, dict_t *src);
+
+#endif /* _GLUSTERD_MGMT_H_ */
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index f1e1e0652..1666f5e4d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -37,6 +37,7 @@
 #include "glusterd-store.h"
 #include "glusterd-hooks.h"
 #include "glusterd-volgen.h"
+#include "glusterd-locks.h"
 #include "syscall.h"
 #include "cli1-xdr.h"
 #include "common-utils.h"
@@ -67,6 +68,165 @@
 static struct list_head gd_op_sm_queue;
 pthread_mutex_t       gd_op_sm_lock;
 glusterd_op_info_t    opinfo = {{0},};
+uuid_t                global_txn_id = {"\0"}; /* To be used in
+                                               * heterogeneous
+                                               * cluster with no
+                                               * transaction ids */
+
+static dict_t *txn_opinfo;
+
+struct txn_opinfo_object_ {
+        glusterd_op_info_t    opinfo;
+};
+typedef struct txn_opinfo_object_ txn_opinfo_obj;
+
+int32_t
+glusterd_txn_opinfo_dict_init ()
+{
+        int32_t ret = -1;
+
+        txn_opinfo = dict_new ();
+        if (!txn_opinfo) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+glusterd_txn_opinfo_dict_fini ()
+{
+        if (txn_opinfo)
+                dict_destroy (txn_opinfo);
+}
+
+void
+glusterd_txn_opinfo_init (glusterd_op_info_t  *opinfo,
+                          glusterd_op_sm_state_info_t *state,
+                          glusterd_op_t *op,
+                          dict_t *op_ctx,
+                          rpcsvc_request_t *req)
+{
+        GF_ASSERT (opinfo);
+
+        if (state)
+                opinfo->state = *state;
+
+        if (op)
+                opinfo->op = *op;
+
+        opinfo->op_ctx = dict_ref(op_ctx);
+
+        if (req)
+                opinfo->req = req;
+
+        return;
+}
+
+int32_t
+glusterd_get_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo)
+{
+        int32_t          ret         = -1;
+        txn_opinfo_obj  *opinfo_obj  = NULL;
+
+        if (!txn_id || !opinfo) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Empty transaction id or opinfo received.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_bin(txn_opinfo, uuid_utoa (*txn_id),
+                           (void **) &opinfo_obj);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to get transaction opinfo");
+                goto out;
+        }
+
+        (*opinfo) = opinfo_obj->opinfo;
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_set_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo)
+{
+        int32_t          ret        = -1;
+        txn_opinfo_obj  *opinfo_obj = NULL;
+
+        if (!txn_id) {
+                gf_log ("", GF_LOG_ERROR, "Empty transaction id received.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_bin(txn_opinfo, uuid_utoa (*txn_id),
+                           (void **) &opinfo_obj);
+        if (ret) {
+                opinfo_obj = GF_CALLOC (1, sizeof(txn_opinfo_obj),
+                                        gf_common_mt_txn_opinfo_obj_t);
+                if (!opinfo_obj) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_set_bin(txn_opinfo, uuid_utoa (*txn_id), opinfo_obj,
+                                   sizeof(txn_opinfo_obj));
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to set opinfo for transaction ID : %s",
+                                uuid_utoa (*txn_id));
+                        goto out;
+                }
+        }
+
+        opinfo_obj->opinfo = (*opinfo);
+
+        ret = 0;
+out:
+        if (ret)
+                if (opinfo_obj)
+                        GF_FREE (opinfo_obj);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_clear_txn_opinfo (uuid_t *txn_id)
+{
+        int32_t               ret         = -1;
+        glusterd_op_info_t    txn_op_info = {{0},};
+
+        if (!txn_id) {
+                gf_log ("", GF_LOG_ERROR, "Empty transaction id received.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_get_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Transaction opinfo not found");
+                goto out;
+        }
+
+        dict_unref (txn_op_info.op_ctx);
+
+        dict_del(txn_opinfo, uuid_utoa (*txn_id));
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
 static int glusterfs_port = GLUSTERD_DEFAULT_PORT;
 static char *glusterd_op_sm_state_names[] = {
         "Default",
@@ -147,10 +307,10 @@ glusterd_is_volume_started (glusterd_volinfo_t  *volinfo)
 }
 
 static int
-glusterd_op_sm_inject_all_acc ()
+glusterd_op_sm_inject_all_acc (uuid_t *txn_id)
 {
         int32_t                 ret = -1;
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC, txn_id, NULL);
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
@@ -235,20 +395,20 @@ glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickin
                 brick_req->name = gf_strdup (name);
 
                 break;
-
-#ifdef HAVE_BD_XLATOR
-        case GD_OP_BD_OP:
-        {
+        case GD_OP_SNAP:
                 brick_req = GF_CALLOC (1, sizeof (*brick_req),
                                        gf_gld_mt_mop_brick_req_t);
                 if (!brick_req)
                         goto out;
 
-                brick_req->op = GLUSTERD_BRICK_BD_OP;
-                brick_req->name = "";
-        }
+                brick_req->op = GLUSTERD_VOLUME_BARRIER_OP;
+                ret = dict_get_str (dict, "volname", &volname);
+                if (ret)
+                        goto out;
+                snprintf (name, 1024, "%s-server",volname);
+                brick_req->name = gf_strdup (name);
+
                 break;
-#endif
         default:
                 goto out;
         break;
@@ -416,11 +576,17 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
         if (!val_dict)
                 goto out;
 
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
         /* Check if we can support the required op-version
          * This check is not done on the originator glusterd. The originator
          * glusterd sets this value.
          */
-        origin_glusterd = is_origin_glusterd ();
+        origin_glusterd = is_origin_glusterd (dict);
 
         if (!origin_glusterd) {
                 /* Check for v3.3.x origin glusterd */
@@ -1101,14 +1267,17 @@ _delete_reconfig_opt (dict_t *this, char *key, data_t *value, void *data)
         GF_ASSERT (data);
         is_force = (int32_t*)data;
 
-        if (*is_force != 1 &&
-            (_gf_true == glusterd_check_voloption_flags (key,
-                                                         OPT_FLAG_FORCE))) {
+        if (*is_force != 1) {
+                if (_gf_true == glusterd_check_voloption_flags (key,
+                                                         OPT_FLAG_FORCE)) {
                 /* indicate to caller that we don't set the option
                  * due to being protected
                  */
-                *is_force = -1;
-                goto out;
+                        *is_force = *is_force | GD_OP_PROTECTED;
+                        goto out;
+                } else {
+                        *is_force = *is_force | GD_OP_UNPROTECTED;
+                }
         }
 
         gf_log ("", GF_LOG_DEBUG, "deleting dict with key=%s,value=%s",
@@ -1277,7 +1446,7 @@ out:
 }
 
 static int
-glusterd_op_reset_volume (dict_t *dict, char **op_errstr)
+glusterd_op_reset_volume (dict_t *dict, char **op_rspstr)
 {
         glusterd_volinfo_t      *volinfo    = NULL;
         int                     ret         = -1;
@@ -1332,10 +1501,18 @@ glusterd_op_reset_volume (dict_t *dict, char **op_errstr)
                 quorum_action = _gf_true;
 
         ret = glusterd_options_reset (volinfo, key, &is_force);
-        if (is_force == -1) {
-                ret = -1;
-                gf_asprintf(op_errstr, "'%s' is protected. To reset use 'force'.",
-                            key);
+        if (ret == -1) {
+                gf_asprintf(op_rspstr, "Volume reset : failed");
+        } else if (is_force & GD_OP_PROTECTED) {
+                if (is_force & GD_OP_UNPROTECTED) {
+                        gf_asprintf (op_rspstr, "All unprotected fields were"
+                                     " reset. To reset the protected fields,"
+                                     " use 'force'.");
+                } else {
+                        ret = -1;
+                        gf_asprintf (op_rspstr, "'%s' is protected. To reset"
+                                     " use 'force'.", key);
+                }
         }
 
 out:
@@ -1365,14 +1542,25 @@ glusterd_stop_bricks (glusterd_volinfo_t *volinfo)
 int
 glusterd_start_bricks (glusterd_volinfo_t *volinfo)
 {
-        glusterd_brickinfo_t                    *brickinfo = NULL;
+        int                      ret            = -1;
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+
+        GF_ASSERT (volinfo);
 
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (glusterd_brick_start (volinfo, brickinfo, _gf_false))
-                        return -1;
+                ret = glusterd_brick_start (volinfo, brickinfo, _gf_false);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "Failed to start %s:%s for %s",
+                                brickinfo->hostname, brickinfo->path,
+                                volinfo->volname);
+                        goto out;
+                }
         }
 
-        return 0;
+        ret = 0;
+out:
+        return ret;
 }
 
 static int
@@ -1475,6 +1663,7 @@ glusterd_op_set_volume (dict_t *dict)
         char                                     str[50] = {0, };
         char                                    *op_errstr = NULL;
         gf_boolean_t                             global_opt    = _gf_false;
+        gf_boolean_t                             global_opts_set = _gf_false;
         glusterd_volinfo_t                      *voliter = NULL;
         int32_t                                  dict_count = 0;
         gf_boolean_t                             check_op_version = _gf_false;
@@ -1536,7 +1725,6 @@ glusterd_op_set_volume (dict_t *dict)
 
         for (count = 1; ret != -1 ; count++) {
 
-                global_opt = _gf_false;
                 sprintf (str, "key%d", count);
                 ret = dict_get_str (dict, str, &key);
                 if (ret)
@@ -1584,8 +1772,11 @@ glusterd_op_set_volume (dict_t *dict)
                         }
                 }
 
-                if (glusterd_check_globaloption (key))
+                global_opt = _gf_false;
+                if (glusterd_check_globaloption (key)) {
                         global_opt = _gf_true;
+                        global_opts_set = _gf_true;
+                }
 
                 if (!global_opt)
                         value = gf_strdup (value);
@@ -1641,7 +1832,7 @@ glusterd_op_set_volume (dict_t *dict)
                 }
         }
 
-        if (!global_opt) {
+        if (!global_opts_set) {
                 gd_update_volume_op_versions (volinfo);
                 ret = glusterd_create_volfiles_and_notify_services (volinfo);
                 if (ret) {
@@ -1901,6 +2092,105 @@ out:
 }
 
 static int
+_add_brick_name_to_dict (dict_t *dict, char *key, glusterd_brickinfo_t *brick)
+{
+        int     ret = -1;
+        char    tmp[1024] = {0,};
+        char    *brickname = NULL;
+        xlator_t *this = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (key);
+        GF_ASSERT (brick);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        snprintf (tmp, sizeof (tmp), "%s:%s", brick->hostname, brick->path);
+        brickname = gf_strdup (tmp);
+        if (!brickname) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to dup brick name");
+                goto out;
+        }
+
+        ret = dict_set_dynstr (dict, key, brickname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to add brick name to dict");
+                goto out;
+        }
+        brickname = NULL;
+out:
+        if (brickname)
+                GF_FREE (brickname);
+        return ret;
+}
+
+static int
+_add_remove_bricks_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo,
+                            char *prefix)
+{
+        int             ret = -1;
+        int             count = 0;
+        int             i = 0;
+        char            brick_key[1024] = {0,};
+        char            dict_key[1024] ={0,};
+        char            *brick = NULL;
+        xlator_t        *this = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (volinfo);
+        GF_ASSERT (prefix);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_int32 (volinfo->rebal.dict, "count", &count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get brick count");
+                goto out;
+        }
+
+        snprintf (dict_key, sizeof (dict_key), "%s.count", prefix);
+        ret = dict_set_int32 (dict, dict_key, count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set brick count in dict");
+                goto out;
+        }
+
+        for (i = 1; i <= count; i++) {
+                memset (brick_key, 0, sizeof (brick_key));
+                snprintf (brick_key, sizeof (brick_key), "brick%d", i);
+
+                ret = dict_get_str (volinfo->rebal.dict, brick_key, &brick);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get %s", brick_key);
+                        goto out;
+                }
+
+                memset (dict_key, 0, sizeof (dict_key));
+                snprintf (dict_key, sizeof (dict_key), "%s.%s", prefix,
+                          brick_key);
+                ret = dict_set_str (dict, dict_key, brick);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add brick to dict");
+                        goto out;
+                }
+                brick = NULL;
+        }
+
+out:
+        return ret;
+}
+
+/* This adds the respective task-id and all available parameters of a task into
+ * a dictionary
+ */
+static int
 _add_task_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo, int op, int index)
 {
 
@@ -1917,13 +2207,34 @@ _add_task_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo, int op, int index)
         GF_ASSERT (this);
 
         switch (op) {
-        case GD_OP_REBALANCE:
         case GD_OP_REMOVE_BRICK:
+                snprintf (key, sizeof (key), "task%d", index);
+                ret = _add_remove_bricks_to_dict (dict, volinfo, key);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add remove bricks to dict");
+                        goto out;
+                }
+        case GD_OP_REBALANCE:
                 uuid_str = gf_strdup (uuid_utoa (volinfo->rebal.rebalance_id));
                 status = volinfo->rebal.defrag_status;
                 break;
 
         case GD_OP_REPLACE_BRICK:
+                snprintf (key, sizeof (key), "task%d.src-brick", index);
+                ret = _add_brick_name_to_dict (dict, key,
+                                               volinfo->rep_brick.src_brick);
+                if (ret)
+                        goto out;
+                memset (key, 0, sizeof (key));
+
+                snprintf (key, sizeof (key), "task%d.dst-brick", index);
+                ret = _add_brick_name_to_dict (dict, key,
+                                               volinfo->rep_brick.dst_brick);
+                if (ret)
+                        goto out;
+                memset (key, 0, sizeof (key));
+
                 uuid_str = gf_strdup (uuid_utoa (volinfo->rep_brick.rb_id));
                 status = volinfo->rep_brick.rb_status;
                 break;
@@ -1936,8 +2247,7 @@ _add_task_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo, int op, int index)
         }
 
         snprintf (key, sizeof (key), "task%d.type", index);
-        ret = dict_set_str (dict, key,
-                            (char *)gd_op_list[op]);
+        ret = dict_set_str (dict, key, (char *)gd_op_list[op]);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "Error setting task type in dict");
@@ -1947,7 +2257,6 @@ _add_task_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo, int op, int index)
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "task%d.id", index);
 
-
         if (!uuid_str)
                 goto out;
         ret = dict_set_dynstr (dict, key, uuid_str);
@@ -1974,6 +2283,50 @@ out:
 }
 
 static int
+glusterd_aggregate_task_status (dict_t *rsp_dict, glusterd_volinfo_t *volinfo)
+{
+        int        ret   = -1;
+        int        tasks = 0;
+        xlator_t  *this  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (!uuid_is_null (volinfo->rebal.rebalance_id)) {
+                ret = _add_task_to_dict (rsp_dict, volinfo, volinfo->rebal.op,
+                                         tasks);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add task details to dict");
+                        goto out;
+                }
+                tasks++;
+        }
+
+        if (!uuid_is_null (volinfo->rep_brick.rb_id)) {
+                ret = _add_task_to_dict (rsp_dict, volinfo, GD_OP_REPLACE_BRICK,
+                                         tasks);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add task details to dict");
+                        goto out;
+                }
+                tasks++;
+        }
+
+        ret = dict_set_int32 (rsp_dict, "tasks", tasks);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting tasks count in dict");
+                goto out;
+        }
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static int
 glusterd_op_status_volume (dict_t *dict, char **op_errstr,
                            dict_t *rsp_dict)
 {
@@ -1993,7 +2346,6 @@ glusterd_op_status_volume (dict_t *dict, char **op_errstr,
         gf_boolean_t            nfs_disabled    = _gf_false;
         gf_boolean_t            shd_enabled     = _gf_true;
         gf_boolean_t            origin_glusterd = _gf_false;
-        int                     tasks           = 0;
 
         this = THIS;
         GF_ASSERT (this);
@@ -2003,13 +2355,13 @@ glusterd_op_status_volume (dict_t *dict, char **op_errstr,
 
         GF_ASSERT (dict);
 
-        origin_glusterd = is_origin_glusterd ();
+        origin_glusterd = is_origin_glusterd (dict);
 
         ret = dict_get_uint32 (dict, "cmd", &cmd);
         if (ret)
                 goto out;
 
-        if (is_origin_glusterd ()) {
+        if (origin_glusterd) {
                 ret = 0;
                 if ((cmd & GF_CLI_STATUS_ALL)) {
                         ret = glusterd_get_all_volnames (rsp_dict);
@@ -2076,6 +2428,10 @@ glusterd_op_status_volume (dict_t *dict, char **op_errstr,
                                                            brick_index);
                 node_count++;
 
+        } else if ((cmd & GF_CLI_STATUS_TASKS) != 0) {
+                ret = glusterd_aggregate_task_status (rsp_dict, volinfo);
+                goto out;
+
         } else {
                 list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                         brick_index++;
@@ -2157,31 +2513,10 @@ glusterd_op_status_volume (dict_t *dict, char **op_errstr,
             !origin_glusterd)
                 goto out;
 
-        if (!uuid_is_null (volinfo->rebal.rebalance_id)) {
-                ret = _add_task_to_dict (rsp_dict, volinfo, volinfo->rebal.op,
-                                         tasks);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to add task details to dict");
-                        goto out;
-                }
-                tasks++;
-        }
-        if (!uuid_is_null (volinfo->rep_brick.rb_id)) {
-                ret = _add_task_to_dict (rsp_dict, volinfo, GD_OP_REPLACE_BRICK,
-                                         tasks);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to add task details to dict");
-                        goto out;
-                }
-                tasks++;
-        }
-
-        ret = dict_set_int32 (rsp_dict, "tasks", tasks);
+        ret = glusterd_aggregate_task_status (rsp_dict, volinfo);
         if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error setting tasks count in dict");
+                goto out;
+        ret = 0;
 
 out:
         gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
@@ -2208,6 +2543,7 @@ glusterd_op_ac_send_lock (glusterd_op_sm_event_t *event, void *ctx)
         xlator_t             *this     = NULL;
         glusterd_peerinfo_t  *peerinfo = NULL;
         uint32_t             pending_count = 0;
+        dict_t               *dict     = NULL;
 
         this = THIS;
         priv = this->private;
@@ -2222,27 +2558,60 @@ glusterd_op_ac_send_lock (glusterd_op_sm_event_t *event, void *ctx)
                     (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
                         continue;
 
-                proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_LOCK];
-                if (proc->fn) {
-                        ret = proc->fn (NULL, this, peerinfo);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING, "Failed to "
-                                        "send lock request for operation "
-                                        "'Volume %s' to peer %s",
-                                        gd_op_list[opinfo.op],
-                                        peerinfo->hostname);
-                                continue;
+                /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
+                if (priv->op_version < 3) {
+                        proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_LOCK];
+                        if (proc->fn) {
+                                ret = proc->fn (NULL, this, peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send lock request "
+                                                "for operation 'Volume %s' to "
+                                                "peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        continue;
+                                }
+                                pending_count++;
+                        }
+                } else {
+                        dict = glusterd_op_get_ctx ();
+                        dict_ref (dict);
+
+                        proc = &peerinfo->mgmt_v3->proctable
+                                                   [GLUSTERD_MGMT_V3_LOCK];
+                        if (proc->fn) {
+                                ret = dict_set_static_ptr (dict, "peerinfo",
+                                                           peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "failed to set peerinfo");
+                                        dict_unref (dict);
+                                        goto out;
+                                }
+
+                                ret = proc->fn (NULL, this, dict);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send mgmt_v3 lock "
+                                                "request for operation "
+                                                "'Volume %s' to peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        dict_unref (dict);
+                                        continue;
+                                }
+                                pending_count++;
                         }
-                        pending_count++;
                 }
         }
 
         opinfo.pending_count = pending_count;
         if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_all_acc ();
+                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
 
+out:
         gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
-
         return ret;
 }
 
@@ -2255,17 +2624,12 @@ glusterd_op_ac_send_unlock (glusterd_op_sm_event_t *event, void *ctx)
         xlator_t             *this     = NULL;
         glusterd_peerinfo_t  *peerinfo = NULL;
         uint32_t             pending_count = 0;
+        dict_t               *dict     = NULL;
 
         this = THIS;
         priv = this->private;
         GF_ASSERT (priv);
 
-        /*ret = glusterd_unlock (MY_UUID);
-
-        if (ret)
-                goto out;
-        */
-
         list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
                 GF_ASSERT (peerinfo);
 
@@ -2275,29 +2639,62 @@ glusterd_op_ac_send_unlock (glusterd_op_sm_event_t *event, void *ctx)
                     (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
                         continue;
 
-                proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_UNLOCK];
-                if (proc->fn) {
-                        ret = proc->fn (NULL, this, peerinfo);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING, "Failed to "
-                                        "send unlock request for operation "
-                                        "'Volume %s' to peer %s",
-                                        gd_op_list[opinfo.op],
-                                        peerinfo->hostname);
-                                continue;
+                /* Based on the op_version, release the *
+                 * cluster or mgmt_v3 lock */
+                if (priv->op_version < 3) {
+                        proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_UNLOCK];
+                        if (proc->fn) {
+                                ret = proc->fn (NULL, this, peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send unlock request "
+                                                "for operation 'Volume %s' to "
+                                                "peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        continue;
+                                }
+                                pending_count++;
+                        }
+                } else {
+                        dict = glusterd_op_get_ctx ();
+                        dict_ref (dict);
+
+                        proc = &peerinfo->mgmt_v3->proctable
+                                                   [GLUSTERD_MGMT_V3_UNLOCK];
+                        if (proc->fn) {
+                                ret = dict_set_static_ptr (dict, "peerinfo",
+                                                           peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "failed to set peerinfo");
+                                        dict_unref (dict);
+                                        goto out;
+                                }
+
+                                ret = proc->fn (NULL, this, dict);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send mgmt_v3 unlock "
+                                                "request for operation "
+                                                "'Volume %s' to peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        dict_unref (dict);
+                                        continue;
+                                }
+                                pending_count++;
                         }
-                        pending_count++;
                 }
         }
 
         opinfo.pending_count = pending_count;
         if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_all_acc ();
+                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
 
+out:
         gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
-
         return ret;
-
 }
 
 static int
@@ -2309,7 +2706,8 @@ glusterd_op_ac_ack_drain (glusterd_op_sm_event_t *event, void *ctx)
                 opinfo.pending_count--;
 
         if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, NULL);
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                                   &event->txn_id, NULL);
 
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
@@ -2325,43 +2723,94 @@ glusterd_op_ac_send_unlock_drain (glusterd_op_sm_event_t *event, void *ctx)
 static int
 glusterd_op_ac_lock (glusterd_op_sm_event_t *event, void *ctx)
 {
-        glusterd_op_lock_ctx_t   *lock_ctx = NULL;
-        int32_t                  ret = 0;
+        int32_t                         ret             = 0;
+        char                           *volname         = NULL;
+        glusterd_op_lock_ctx_t         *lock_ctx        = NULL;
+        glusterd_conf_t                *priv            = NULL;
+        xlator_t                       *this            = NULL;
 
         GF_ASSERT (event);
         GF_ASSERT (ctx);
 
+        this = THIS;
+        priv = this->private;
+
         lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
 
-        ret = glusterd_lock (lock_ctx->uuid);
+        /* If the req came from a node running on older op_version
+         * the dict won't be present. Based on it acquiring a cluster
+         * or mgmt_v3 lock */
+        if (lock_ctx->dict == NULL) {
+                ret = glusterd_lock (lock_ctx->uuid);
+                glusterd_op_lock_send_resp (lock_ctx->req, ret);
+        } else {
+                ret = dict_get_str (lock_ctx->dict, "volname", &volname);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire volname");
+                else {
+                        ret = glusterd_mgmt_v3_lock (volname, lock_ctx->uuid,
+                                                     "vol");
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to acquire lock for %s",
+                                        volname);
+                }
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Lock Returned %d", ret);
+                glusterd_op_mgmt_v3_lock_send_resp (lock_ctx->req,
+                                                    &event->txn_id, ret);
 
-        glusterd_op_lock_send_resp (lock_ctx->req, ret);
+                dict_unref (lock_ctx->dict);
+        }
 
+        gf_log (THIS->name, GF_LOG_DEBUG, "Lock Returned %d", ret);
         return ret;
 }
 
 static int
 glusterd_op_ac_unlock (glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                      ret = 0;
-        glusterd_op_lock_ctx_t   *lock_ctx = NULL;
-        xlator_t                 *this = NULL;
-        glusterd_conf_t          *priv = NULL;
+        int32_t                         ret             = 0;
+        char                           *volname         = NULL;
+        glusterd_op_lock_ctx_t         *lock_ctx        = NULL;
+        glusterd_conf_t                *priv            = NULL;
+        xlator_t                       *this            = NULL;
+
 
         GF_ASSERT (event);
         GF_ASSERT (ctx);
 
         this = THIS;
         priv = this->private;
+
         lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
 
-        ret = glusterd_unlock (lock_ctx->uuid);
+        /* If the req came from a node running on older op_version
+         * the dict won't be present. Based on it releasing the cluster
+         * or mgmt_v3 lock */
+        if (lock_ctx->dict == NULL) {
+                ret = glusterd_unlock (lock_ctx->uuid);
+                glusterd_op_unlock_send_resp (lock_ctx->req, ret);
+        } else {
+                ret = dict_get_str (lock_ctx->dict, "volname", &volname);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire volname");
+                else {
+                        ret = glusterd_mgmt_v3_unlock (volname, lock_ctx->uuid,
+                                                       "vol");
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to release lock for %s", volname);
+                }
 
-        gf_log (this->name, GF_LOG_DEBUG, "Unlock Returned %d", ret);
+                glusterd_op_mgmt_v3_unlock_send_resp (lock_ctx->req,
+                                                      &event->txn_id, ret);
 
-        glusterd_op_unlock_send_resp (lock_ctx->req, ret);
+                dict_unref (lock_ctx->dict);
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Unlock Returned %d", ret);
 
         if (priv->pending_quorum_action)
                 glusterd_do_quorum_action ();
@@ -2399,7 +2848,8 @@ glusterd_op_ac_rcvd_lock_acc (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC,
+                                           &event->txn_id, NULL);
 
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
@@ -2603,9 +3053,6 @@ glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx)
                 case GD_OP_STATEDUMP_VOLUME:
                 case GD_OP_CLEARLOCKS_VOLUME:
                 case GD_OP_DEFRAG_BRICK_VOLUME:
-#ifdef HAVE_BD_XLATOR
-                case GD_OP_BD_OP:
-#endif
                         {
                                 ret = dict_get_str (dict, "volname", &volname);
                                 if (ret) {
@@ -2834,7 +3281,8 @@ out:
         if (dict)
                 dict_unref (dict);
         if (ret) {
-                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, NULL);
+                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                             &event->txn_id, NULL);
                 opinfo.op_ret = ret;
         }
 
@@ -2843,7 +3291,7 @@ out:
                 opinfo.pending_count);
 
         if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_all_acc ();
+                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
 
         gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
 
@@ -2852,10 +3300,10 @@ out:
 }
 
 static int32_t
-glusterd_op_start_rb_timer (dict_t *dict)
+glusterd_op_start_rb_timer (dict_t *dict, uuid_t *txn_id)
 {
         int32_t         op = 0;
-        struct timeval  timeout = {0, };
+        struct timespec timeout = {0, };
         glusterd_conf_t *priv = NULL;
         int32_t         ret = -1;
         dict_t          *rb_ctx = NULL;
@@ -2871,12 +3319,12 @@ glusterd_op_start_rb_timer (dict_t *dict)
         }
 
         if (op != GF_REPLACE_OP_START) {
-                ret = glusterd_op_sm_inject_all_acc ();
+                ret = glusterd_op_sm_inject_all_acc (txn_id);
                 goto out;
         }
 
         timeout.tv_sec  = 5;
-        timeout.tv_usec = 0;
+        timeout.tv_nsec = 0;
 
 
         rb_ctx = dict_copy (dict, rb_ctx);
@@ -2886,6 +3334,17 @@ glusterd_op_start_rb_timer (dict_t *dict)
                 ret = -1;
                 goto out;
         }
+
+        ret = dict_set_bin (rb_ctx, "transaction_id",
+                            txn_id, sizeof(*txn_id));
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                       "Failed to set transaction id.");
+                goto out;
+        } else
+               gf_log ("", GF_LOG_DEBUG,
+                        "transaction_id = %s", uuid_utoa (*txn_id));
+
         priv->timer = gf_timer_call_after (THIS->ctx, timeout,
                                            glusterd_do_replace_brick,
                                            (void *) rb_ctx);
@@ -2954,6 +3413,97 @@ out:
         return ret;
 }
 
+static int
+reassign_defrag_status (dict_t *dict, char *key, gf_defrag_status_t *status)
+{
+        int ret = 0;
+
+        if (!*status)
+                return ret;
+
+        switch (*status) {
+        case GF_DEFRAG_STATUS_STARTED:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED;
+                break;
+
+        case GF_DEFRAG_STATUS_STOPPED:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED;
+                break;
+
+        case GF_DEFRAG_STATUS_COMPLETE:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE;
+                break;
+
+        case GF_DEFRAG_STATUS_FAILED:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED;
+                break;
+        default:
+                break;
+         }
+
+        ret = dict_set_int32(dict, key, *status);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to reset defrag %s in dict", key);
+
+        return ret;
+}
+
+/* Check and reassign the defrag_status enum got from the rebalance process
+ * of all peers so that the rebalance-status CLI command can display if a
+ * full-rebalance or just a fix-layout was carried out.
+ */
+static int
+glusterd_op_check_peer_defrag_status (dict_t *dict, int count)
+{
+        glusterd_volinfo_t *volinfo  = NULL;
+        gf_defrag_status_t status    = GF_DEFRAG_STATUS_NOT_STARTED;
+        char               key[256]  = {0,};
+        char               *volname  = NULL;
+        int                ret       = -1;
+        int                i         = 1;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_WARNING, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_WARNING, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                goto out;
+        }
+
+        if (volinfo->rebal.defrag_cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+                /* Fix layout was not issued; we don't need to reassign
+                   the status */
+                ret = 0;
+                goto out;
+        }
+
+        do {
+                memset (key, 0, 256);
+                snprintf (key, 256, "status-%d", i);
+                ret = dict_get_int32 (dict, key, (int32_t *)&status);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to get defrag %s", key);
+                        goto out;
+                }
+                ret = reassign_defrag_status (dict, key, &status);
+                if (ret)
+                        goto out;
+                i++;
+        } while (i <= count);
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
 /* This function is used to modify the op_ctx dict before sending it back
  * to cli. This is useful in situations like changing the peer uuids to
  * hostnames etc.
@@ -3057,12 +3607,38 @@ glusterd_op_modify_op_ctx (glusterd_op_t op, void *ctx)
                         goto out;
                 }
 
+                /* add 'node-name-%d' into op_ctx with value uuid_str.
+                   this will be used to convert to hostname later */
+                {
+                        char  key[1024];
+                        char *uuid_str = NULL;
+                        int   i;
+
+                        for (i = 1; i <= count; i++) {
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key), "node-uuid-%d", i);
+                                ret = dict_get_str (op_ctx, key, &uuid_str);
+                                if (!ret) {
+                                        memset (key, 0, sizeof (key));
+                                        snprintf (key, sizeof (key),
+                                                  "node-name-%d", i);
+                                        ret = dict_set_str (op_ctx, key,
+                                                            uuid_str);
+                                }
+                        }
+                }
+
                 ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
-                                                                "node-uuid-%d",
+                                                                "node-name-%d",
                                                                 1, (count + 1));
                 if (ret)
                         gf_log (this->name, GF_LOG_WARNING,
                                 "Failed uuid to hostname conversion");
+
+                ret = glusterd_op_check_peer_defrag_status (op_ctx, count);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to reset defrag status for fix-layout");
                 break;
 
         default:
@@ -3215,17 +3791,19 @@ out:
         if (dict)
                 dict_unref (dict);
         if (ret) {
-                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, NULL);
+                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                             &event->txn_id, NULL);
                 opinfo.op_ret = ret;
         }
 
         if (!opinfo.pending_count) {
                 if (op == GD_OP_REPLACE_BRICK) {
-                        ret = glusterd_op_start_rb_timer (op_dict);
+                        ret = glusterd_op_start_rb_timer (op_dict,
+                                                          &event->txn_id);
 
                 } else {
                         glusterd_op_modify_op_ctx (op, NULL);
-                        ret = glusterd_op_sm_inject_all_acc ();
+                        ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
                 }
                 goto err;
         }
@@ -3250,7 +3828,8 @@ glusterd_op_ac_rcvd_stage_op_acc (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_ACC, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_ACC,
+                                           &event->txn_id, NULL);
 
 out:
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
@@ -3271,7 +3850,8 @@ glusterd_op_ac_stage_op_failed (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                           &event->txn_id, NULL);
 
 out:
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
@@ -3292,7 +3872,8 @@ glusterd_op_ac_commit_op_failed (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                           &event->txn_id, NULL);
 
 out:
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
@@ -3335,7 +3916,8 @@ glusterd_op_ac_brick_op_failed (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.brick_pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, ev_ctx->commit_ctx);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                           &event->txn_id, ev_ctx->commit_ctx);
 
 out:
         if (ev_ctx->rsp_dict)
@@ -3377,7 +3959,7 @@ glusterd_op_ac_rcvd_commit_op_acc (glusterd_op_sm_event_t *event, void *ctx)
                         goto out;
                 }
 
-                ret = glusterd_op_start_rb_timer (op_ctx);
+                ret = glusterd_op_start_rb_timer (op_ctx, &event->txn_id);
                 if (ret) {
                         gf_log (this->name, GF_LOG_ERROR, "Couldn't start "
                                 "replace-brick operation.");
@@ -3392,10 +3974,12 @@ glusterd_op_ac_rcvd_commit_op_acc (glusterd_op_sm_event_t *event, void *ctx)
 out:
         if (commit_ack_inject) {
                 if (ret)
-                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, NULL);
+                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                                           &event->txn_id, NULL);
                 else if (!opinfo.pending_count) {
                         glusterd_op_modify_op_ctx (op, NULL);
-                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_ACC, NULL);
+                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_ACC,
+                                                           &event->txn_id, NULL);
                 }
                 /*else do nothing*/
         }
@@ -3416,7 +4000,8 @@ glusterd_op_ac_rcvd_unlock_acc (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC, NULL);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC,
+                                           &event->txn_id, NULL);
 
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
@@ -3450,7 +4035,7 @@ glusterd_op_reset_ctx ()
 }
 
 int32_t
-glusterd_op_txn_complete ()
+glusterd_op_txn_complete (uuid_t *txn_id)
 {
         int32_t                 ret = -1;
         glusterd_conf_t         *priv = NULL;
@@ -3460,6 +4045,7 @@ glusterd_op_txn_complete ()
         rpcsvc_request_t        *req = NULL;
         void                    *ctx = NULL;
         char                    *op_errstr = NULL;
+        char                    *volname = NULL;
         xlator_t                *this = NULL;
 
         this = THIS;
@@ -3482,14 +4068,29 @@ glusterd_op_txn_complete ()
         glusterd_op_reset_ctx ();
         glusterd_op_clear_errstr ();
 
-        ret = glusterd_unlock (MY_UUID);
-
-        /* unlock cant/shouldnt fail here!! */
-        if (ret) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Unable to clear local lock, ret: %d", ret);
+        /* Based on the op-version, we release the cluster or mgmt_v3 lock */
+        if (priv->op_version < 3) {
+                ret = glusterd_unlock (MY_UUID);
+                /* unlock cant/shouldnt fail here!! */
+                if (ret)
+                        gf_log (this->name, GF_LOG_CRITICAL,
+                                "Unable to clear local lock, ret: %d", ret);
+                else
+                        gf_log (this->name, GF_LOG_DEBUG, "Cleared local lock");
         } else {
-                gf_log (this->name, GF_LOG_DEBUG, "Cleared local lock");
+                ret = dict_get_str (ctx, "volname", &volname);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire volname");
+
+                if (volname) {
+                        ret = glusterd_mgmt_v3_unlock (volname, MY_UUID,
+                                                       "vol");
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to release lock for %s",
+                                        volname);
+                }
         }
 
         ret = glusterd_op_send_cli_response (op, op_ret,
@@ -3508,6 +4109,13 @@ glusterd_op_txn_complete ()
 
         if (priv->pending_quorum_action)
                 glusterd_do_quorum_action ();
+
+        /* Clearing the transaction opinfo */
+        ret = glusterd_clear_txn_opinfo (txn_id);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to clear transaction's opinfo");
+
         gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
@@ -3519,7 +4127,7 @@ glusterd_op_ac_unlocked_all (glusterd_op_sm_event_t *event, void *ctx)
 
         GF_ASSERT (event);
 
-        ret = glusterd_op_txn_complete ();
+        ret = glusterd_op_txn_complete (&event->txn_id);
 
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
 
@@ -3536,6 +4144,7 @@ glusterd_op_ac_stage_op (glusterd_op_sm_event_t *event, void *ctx)
         char                    *op_errstr = NULL;
         dict_t                  *dict = NULL;
         xlator_t                *this = NULL;
+        uuid_t                  *txn_id = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -3561,6 +4170,19 @@ glusterd_op_ac_stage_op (glusterd_op_sm_event_t *event, void *ctx)
                         status);
         }
 
+        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+
+        if (txn_id)
+                uuid_copy (*txn_id, event->txn_id);
+        else
+                gf_log (this->name, GF_LOG_ERROR, "Out of Memory");
+
+        ret = dict_set_bin (rsp_dict, "transaction_id",
+                            txn_id, sizeof(*txn_id));
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to set transaction id.");
+
         ret = glusterd_op_stage_send_resp (req_ctx->req, req_ctx->op,
                                            status, op_errstr, rsp_dict);
 
@@ -3625,6 +4247,7 @@ glusterd_op_ac_commit_op (glusterd_op_sm_event_t *event, void *ctx)
         dict_t                   *dict       = NULL;
         dict_t                   *rsp_dict   = NULL;
         xlator_t                 *this       = NULL;
+        uuid_t                   *txn_id     = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -3654,10 +4277,22 @@ glusterd_op_ac_commit_op (glusterd_op_sm_event_t *event, void *ctx)
                         "'Volume %s' failed: %d", gd_op_list[req_ctx->op],
                         status);
 
+        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+
+        if (txn_id)
+                uuid_copy (*txn_id, event->txn_id);
+        else
+                gf_log (this->name, GF_LOG_ERROR, "Out of Memory");
+
+        ret = dict_set_bin (rsp_dict, "transaction_id",
+                            txn_id, sizeof(*txn_id));
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to set transaction id.");
+
         ret = glusterd_op_commit_send_resp (req_ctx->req, req_ctx->op,
                                             status, op_errstr, rsp_dict);
 
-        glusterd_op_fini_ctx ();
         if (op_errstr && (strcmp (op_errstr, "")))
                 GF_FREE (op_errstr);
 
@@ -3686,7 +4321,6 @@ glusterd_op_ac_send_commit_failed (glusterd_op_sm_event_t *event, void *ctx)
                                             opinfo.op_ret, opinfo.op_errstr,
                                             op_ctx);
 
-        glusterd_op_fini_ctx ();
         if (opinfo.op_errstr && (strcmp (opinfo.op_errstr, ""))) {
                 GF_FREE (opinfo.op_errstr);
                 opinfo.op_errstr = NULL;
@@ -3808,11 +4442,6 @@ glusterd_op_stage_validate (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         ret = glusterd_op_stage_clearlocks_volume (dict,
                                                                    op_errstr);
                         break;
-#ifdef HAVE_BD_XLATOR
-                case GD_OP_BD_OP:
-                        ret = glusterd_op_stage_bd (dict, op_errstr);
-                        break;
-#endif
 
                 case GD_OP_COPY_FILE:
                         ret = glusterd_op_stage_copy_file (dict, op_errstr);
@@ -3924,11 +4553,6 @@ glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
                         ret = glusterd_op_clearlocks_volume (dict, op_errstr,
                                                              rsp_dict);
                         break;
-#ifdef HAVE_BD_XLATOR
-                case GD_OP_BD_OP:
-                        ret = 0;
-                        break;
-#endif
 
                 case GD_OP_COPY_FILE:
                         ret = glusterd_op_copy_file (dict, op_errstr);
@@ -3951,6 +4575,7 @@ glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
         return ret;
 }
 
+
 static int
 glusterd_bricks_select_stop_volume (dict_t *dict, char **op_errstr,
                                     struct list_head *selected)
@@ -4267,24 +4892,95 @@ out:
 }
 
 int
+get_replica_index_for_per_replica_cmd (glusterd_volinfo_t *volinfo,
+                                       dict_t *dict) {
+        int                     ret = 0;
+        char                    *hostname = NULL;
+        char                    *path = NULL;
+        int                     index = 0;
+        glusterd_brickinfo_t   *brickinfo = NULL;
+        int                     cmd_replica_index = -1;
+        int                     replica_count = -1;
+
+
+        if (!dict)  {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "per-replica-cmd-hostname", &hostname);
+        if (ret)
+                goto out;
+        ret = dict_get_str (dict, "per-replica-cmd-path", &path);
+        if (ret)
+                goto out;
+
+        replica_count = volinfo->replica_count;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_is_null (brickinfo->uuid))
+                        (void)glusterd_resolve_brick (brickinfo);
+                if (!strcmp (brickinfo->path,  path) &&
+                    !strcmp (brickinfo->hostname, hostname)) {
+                        cmd_replica_index = index/(replica_count);
+                        goto out;
+                }
+                index++;
+        }
+
+
+out:
+        if (ret)
+                cmd_replica_index = -1;
+
+        return cmd_replica_index;
+}
+
+int
 _select_rxlators_with_local_bricks (xlator_t *this, glusterd_volinfo_t *volinfo,
-                                    dict_t *dict)
+                                    dict_t *dict, cli_cmd_type type)
 {
         glusterd_brickinfo_t    *brickinfo = NULL;
         glusterd_conf_t         *priv   = NULL;
-        int                     index = 1;
+        int                     index = 0;
         int                     rxlator_count = 0;
         int                     replica_count = 0;
         gf_boolean_t            add     = _gf_false;
+        int                     ret = 0;
+        int                     cmd_replica_index = -1;
 
         priv = this->private;
         replica_count = volinfo->replica_count;
+
+        if (type == PER_REPLICA) {
+
+                cmd_replica_index = get_replica_index_for_per_replica_cmd
+                                    (volinfo, dict);
+                if (cmd_replica_index == -1) {
+                        ret = -1;
+                        goto err;
+                }
+        }
+
+        index = 1;
+
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                 if (uuid_is_null (brickinfo->uuid))
                         (void)glusterd_resolve_brick (brickinfo);
 
-                if (!uuid_compare (MY_UUID, brickinfo->uuid))
-                        add = _gf_true;
+                switch (type) {
+                        case ALL_REPLICA:
+                                if (!uuid_compare (MY_UUID, brickinfo->uuid))
+                                        add = _gf_true;
+                                break;
+                        case PER_REPLICA:
+                                if (!uuid_compare (MY_UUID, brickinfo->uuid) &&
+                                 ((index-1)/replica_count == cmd_replica_index))
+
+                                                add = _gf_true;
+                                break;
+                }
+
                 if (index % replica_count == 0) {
                         if (add) {
                                 _add_rxlator_to_dict (dict, volinfo->volname,
@@ -4297,6 +4993,10 @@ _select_rxlators_with_local_bricks (xlator_t *this, glusterd_volinfo_t *volinfo,
 
                 index++;
         }
+err:
+        if (ret)
+                rxlator_count = -1;
+
         return rxlator_count;
 }
 
@@ -4337,9 +5037,10 @@ _select_rxlators_for_full_self_heal (xlator_t *this,
         return rxlator_count;
 }
 
-#ifdef HAVE_BD_XLATOR
+
 static int
-glusterd_bricks_select_bd (dict_t *dict, char **op_errstr)
+glusterd_bricks_select_snap (dict_t *dict, char **op_errstr,
+                             struct list_head *selected)
 {
         int                        ret           = -1;
         glusterd_conf_t            *priv         = NULL;
@@ -4357,31 +5058,31 @@ glusterd_bricks_select_bd (dict_t *dict, char **op_errstr)
 
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volname");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get"
+                        " volname");
                 goto out;
         }
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret)
                 goto out;
 
-        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                  gf_gld_mt_pending_node_t);
-        if (!pending_node) {
-                ret = -1;
-                goto out;
-        }
-
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                 brick_index++;
                 if (uuid_compare (brickinfo->uuid, MY_UUID) ||
                     !glusterd_is_brick_started (brickinfo)) {
                         continue;
                 }
+                pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                          gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                        ret = -1;
+                        goto out;
+                }
                 pending_node->node = brickinfo;
                 pending_node->type = GD_NODE_BRICK;
                 pending_node->index = brick_index;
                 list_add_tail (&pending_node->list,
-                               &opinfo.pending_bricks);
+                               selected);
                 pending_node = NULL;
         }
 
@@ -4391,10 +5092,10 @@ out:
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning ret %d", ret);
         return ret;
 }
-#endif
 
 static int
-fill_shd_status_for_local_bricks (dict_t *dict, glusterd_volinfo_t *volinfo)
+fill_shd_status_for_local_bricks (dict_t *dict, glusterd_volinfo_t *volinfo,
+                                  cli_cmd_type type, dict_t *req_dict)
 {
         glusterd_brickinfo_t    *brickinfo = NULL;
         char                    msg[1024] = {0,};
@@ -4403,10 +5104,22 @@ fill_shd_status_for_local_bricks (dict_t *dict, glusterd_volinfo_t *volinfo)
         int                     index = 0;
         int                     ret = 0;
         xlator_t               *this = NULL;
+        int                     cmd_replica_index = -1;
 
         this = THIS;
         snprintf (msg, sizeof (msg), "self-heal-daemon is not running on");
 
+        if (type == PER_REPLICA) {
+                cmd_replica_index = get_replica_index_for_per_replica_cmd
+                                    (volinfo, req_dict);
+                if (cmd_replica_index == -1) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "Could not find the "
+                                "replica index for per replica type command");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
         list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                 if (uuid_is_null (brickinfo->uuid))
                         (void)glusterd_resolve_brick (brickinfo);
@@ -4415,6 +5128,14 @@ fill_shd_status_for_local_bricks (dict_t *dict, glusterd_volinfo_t *volinfo)
                         index++;
                         continue;
                 }
+
+                if (type == PER_REPLICA) {
+                      if (cmd_replica_index != (index/volinfo->replica_count)) {
+                              index++;
+                              continue;
+                        }
+
+                }
                 snprintf (key, sizeof (key), "%d-status",index);
                 snprintf (value, sizeof (value), "%s %s",msg,
                           uuid_utoa(MY_UUID));
@@ -4483,21 +5204,49 @@ glusterd_bricks_select_heal_volume (dict_t *dict, char **op_errstr,
                 goto out;
         }
 
+        switch (heal_op) {
+                case GF_AFR_OP_INDEX_SUMMARY:
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+                if (!glusterd_is_nodesvc_online ("glustershd")) {
+                        if (!rsp_dict) {
+                                gf_log (this->name, GF_LOG_ERROR, "Received "
+                                        "empty ctx.");
+                                goto out;
+                        }
 
-        if (!glusterd_is_nodesvc_online ("glustershd") &&
-            (heal_op == GF_AFR_OP_INDEX_SUMMARY)) {
-
-                if (!rsp_dict) {
-                        gf_log (this->name, GF_LOG_ERROR, "Received empty "
-                                "ctx.");
+                        ret = fill_shd_status_for_local_bricks (rsp_dict,
+                                                                volinfo,
+                                                                ALL_REPLICA,
+                                                                dict);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "fill the shd status for the local "
+                                        "bricks");
                         goto out;
+
                 }
+                break;
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                if (!glusterd_is_nodesvc_online ("glustershd")) {
+                        if (!rsp_dict) {
+                                gf_log (this->name, GF_LOG_ERROR, "Received "
+                                        "empty ctx.");
+                                goto out;
+                        }
+                        ret = fill_shd_status_for_local_bricks (rsp_dict,
+                                                                volinfo,
+                                                                PER_REPLICA,
+                                                                dict);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "fill the shd status for the local"
+                                        " bricks.");
+                        goto out;
 
-                ret = fill_shd_status_for_local_bricks (rsp_dict, volinfo);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to fill the shd"
-                                " status for the local bricks");
-                goto out;
+                }
+                break;
+                default:
+                        break;
         }
 
 
@@ -4507,14 +5256,28 @@ glusterd_bricks_select_heal_volume (dict_t *dict, char **op_errstr,
                                                                      volinfo,
                                                                      dict);
                 break;
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                rxlator_count = _select_rxlators_with_local_bricks (this,
+                                                                   volinfo,
+                                                                   dict,
+                                                                   PER_REPLICA);
+                break;
         default:
                 rxlator_count = _select_rxlators_with_local_bricks (this,
                                                                     volinfo,
-                                                                    dict);
+                                                                    dict,
+                                                                   ALL_REPLICA);
                 break;
         }
         if (!rxlator_count)
                 goto out;
+        if (rxlator_count == -1){
+                gf_log (this->name, GF_LOG_ERROR, "Could not determine the"
+                        "translator count");
+                ret = -1;
+                goto out;
+        }
+
         ret = dict_set_int32 (dict, "count", rxlator_count);
         if (ret)
                 goto out;
@@ -4776,7 +5539,8 @@ glusterd_op_ac_send_brick_op (glusterd_op_sm_event_t *event, void *ctx)
 
         if (!opinfo.pending_count && !opinfo.brick_pending_count) {
                 glusterd_clear_pending_nodes (&opinfo.pending_bricks);
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, req_ctx);
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                                   &event->txn_id, req_ctx);
         }
 
 out:
@@ -4830,7 +5594,8 @@ glusterd_op_ac_rcvd_brick_op_acc (glusterd_op_sm_event_t *event, void *ctx)
         if (opinfo.brick_pending_count > 0)
                 goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, ev_ctx->commit_ctx);
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                           ev_ctx->commit_ctx);
 
 out:
         if (ev_ctx->rsp_dict)
@@ -4882,11 +5647,9 @@ glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr,
                 ret = glusterd_bricks_select_rebalance_volume (dict, op_errstr,
                                                                selected);
                 break;
-#ifdef HAVE_BD_XLATOR
-        case GD_OP_BD_OP:
-                ret = glusterd_bricks_select_bd (dict, op_errstr);
+        case GD_OP_SNAP:
+                ret = glusterd_bricks_select_snap (dict, op_errstr, selected);
                 break;
-#endif
         default:
                 break;
          }
@@ -5208,7 +5971,7 @@ glusterd_op_sm_new_event (glusterd_op_sm_event_type_t event_type,
 
 int
 glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
-                             void *ctx)
+                             uuid_t *txn_id, void *ctx)
 {
         int32_t                 ret = -1;
         glusterd_op_sm_event_t  *event = NULL;
@@ -5223,6 +5986,9 @@ glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
 
         event->ctx = ctx;
 
+        if (txn_id)
+                uuid_copy (event->txn_id, *txn_id);
+
         gf_log (THIS->name, GF_LOG_DEBUG, "Enqueue event: '%s'",
                 glusterd_op_sm_event_name_get (event->event));
         list_add_tail (&event->list, &gd_op_sm_queue);
@@ -5283,6 +6049,7 @@ glusterd_op_sm ()
         glusterd_op_sm_t                *state = NULL;
         glusterd_op_sm_event_type_t     event_type = GD_OP_EVENT_NONE;
         xlator_t                        *this = NULL;
+        glusterd_op_info_t              txn_op_info;
 
         this = THIS;
         GF_ASSERT (this);
@@ -5303,6 +6070,20 @@ glusterd_op_sm ()
                                 "type: '%s'",
                                 glusterd_op_sm_event_name_get(event_type));
 
+                        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s",
+                                uuid_utoa (event->txn_id));
+
+                        ret = glusterd_get_txn_opinfo (&event->txn_id,
+                                                       &txn_op_info);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to get transaction's opinfo");
+                                glusterd_destroy_op_event_ctx (event);
+                                GF_FREE (event);
+                                continue;
+                        } else
+                                opinfo = txn_op_info;
+
                         state = glusterd_op_state_table[opinfo.state.state];
 
                         GF_ASSERT (state);
@@ -5333,8 +6114,27 @@ glusterd_op_sm ()
                                 return ret;
                         }
 
+                        if ((state[event_type].next_state ==
+                             GD_OP_STATE_DEFAULT) &&
+                           (event_type == GD_OP_EVENT_UNLOCK)) {
+                                /* Clearing the transaction opinfo */
+                                ret = glusterd_clear_txn_opinfo(&event->txn_id);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Unable to clear "
+                                                "transaction's opinfo");
+                        } else {
+                                ret = glusterd_set_txn_opinfo (&event->txn_id,
+                                                               &opinfo);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Unable to set "
+                                                "transaction's opinfo");
+                        }
+
                         glusterd_destroy_op_event_ctx (event);
                         GF_FREE (event);
+
                 }
         }
 
@@ -5388,52 +6188,6 @@ glusterd_op_clear_op (glusterd_op_t op)
 }
 
 int32_t
-glusterd_op_init_ctx (glusterd_op_t op)
-{
-        int     ret = 0;
-        dict_t *dict = NULL;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (GD_OP_NONE < op && op < GD_OP_MAX);
-
-        if (_gf_false == glusterd_need_brick_op (op)) {
-                gf_log (this->name, GF_LOG_DEBUG, "Received op: %s, returning",
-                        gd_op_list[op]);
-                goto out;
-        }
-        dict = dict_new ();
-        if (dict == NULL) {
-                ret = -1;
-                goto out;
-        }
-        ret = glusterd_op_set_ctx (dict);
-        if (ret)
-                goto out;
-out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-
-
-int32_t
-glusterd_op_fini_ctx ()
-{
-        dict_t *dict = NULL;
-
-        dict = glusterd_op_get_ctx ();
-        if (dict)
-                dict_unref (dict);
-
-        glusterd_op_reset_ctx ();
-        return 0;
-}
-
-
-
-int32_t
 glusterd_op_free_ctx (glusterd_op_t op, void *ctx)
 {
 
@@ -5459,9 +6213,6 @@ glusterd_op_free_ctx (glusterd_op_t op, void *ctx)
                 case GD_OP_STATEDUMP_VOLUME:
                 case GD_OP_CLEARLOCKS_VOLUME:
                 case GD_OP_DEFRAG_BRICK_VOLUME:
-#ifdef HAVE_BD_XLATOR
-                case GD_OP_BD_OP:
-#endif
                         dict_unref (ctx);
                         break;
                 default:
@@ -5490,4 +6241,3 @@ glusterd_op_sm_init ()
         pthread_mutex_init (&gd_op_sm_lock, NULL);
         return 0;
 }
-
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.h b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
index 321c5c484..cf57b78e0 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
@@ -32,6 +32,8 @@
 #include "protocol-common.h"
 
 #define GD_VOLUME_NAME_MAX 256
+#define GD_OP_PROTECTED    (0x02)
+#define GD_OP_UNPROTECTED  (0x04)
 
 typedef enum glusterd_op_sm_state_ {
         GD_OP_STATE_DEFAULT = 0,
@@ -75,6 +77,7 @@ struct glusterd_op_sm_event_ {
         struct list_head                list;
         void                            *ctx;
         glusterd_op_sm_event_type_t     event;
+        uuid_t                          txn_id;
 };
 
 typedef struct glusterd_op_sm_event_ glusterd_op_sm_event_t;
@@ -117,6 +120,7 @@ typedef struct glusterd_op_log_filename_ctx_ glusterd_op_log_filename_ctx_t;
 
 struct glusterd_op_lock_ctx_ {
         uuid_t                  uuid;
+        dict_t                 *dict;
         rpcsvc_request_t        *req;
 };
 
@@ -162,12 +166,18 @@ typedef struct glusterd_gsync_status_temp {
         glusterd_volinfo_t *volinfo;
         char *node;
 }glusterd_gsync_status_temp_t;
+
+typedef enum cli_cmd_type_ {
+        PER_REPLICA,
+        ALL_REPLICA,
+ } cli_cmd_type;
+
 int
 glusterd_op_sm_new_event (glusterd_op_sm_event_type_t event_type,
                           glusterd_op_sm_event_t **new_event);
 int
 glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
-                             void *ctx);
+                             uuid_t *txn_id, void *ctx);
 
 int
 glusterd_op_sm_init ();
@@ -251,10 +261,7 @@ glusterd_op_init_commit_rsp_dict (glusterd_op_t op);
 
 void
 glusterd_op_modify_op_ctx (glusterd_op_t op, void *op_ctx);
-int32_t
-glusterd_op_init_ctx (glusterd_op_t op);
-int32_t
-glusterd_op_fini_ctx ();
+
 int32_t
 glusterd_volume_stats_read_perf (char *brick_path, int32_t blk_size,
                 int32_t blk_count, double *throughput, double *time);
@@ -278,4 +285,15 @@ glusterd_check_gsync_running (glusterd_volinfo_t *volinfo, gf_boolean_t *flag);
 int
 glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
                                  dict_t *op_ctx);
+int
+glusterd_is_valid_vg (glusterd_brickinfo_t *brick, int check_tag, char *msg);
+
+int32_t
+glusterd_get_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo);
+
+int32_t
+glusterd_set_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo);
+
+int32_t
+glusterd_clear_txn_opinfo (uuid_t *txn_id);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.c b/xlators/mgmt/glusterd/src/glusterd-pmap.c
index 9fdd782fe..a153ca1a9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-pmap.c
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.c
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -52,8 +52,8 @@ pmap_port_isfree (int port)
 }
 
 
-struct pmap_registry *
-pmap_registry_new (void)
+static struct pmap_registry *
+pmap_registry_new (xlator_t *this)
 {
         struct pmap_registry *pmap = NULL;
         int                   i = 0;
@@ -69,8 +69,8 @@ pmap_registry_new (void)
                         pmap->ports[i].type = GF_PMAP_PORT_FOREIGN;
         }
 
-        pmap->base_port = GF_IANA_PRIV_PORTS_START;
-        pmap->last_alloc = GF_IANA_PRIV_PORTS_START;
+        pmap->base_port = pmap->last_alloc =
+                ((glusterd_conf_t *)(this->private))->base_port;
 
         return pmap;
 }
@@ -86,7 +86,7 @@ pmap_registry_get (xlator_t *this)
 
         pmap = priv->pmap;
         if (!pmap) {
-                pmap = pmap_registry_new ();
+                pmap = pmap_registry_new (this);
                 if (!pmap)
                         return NULL;
                 priv->pmap = pmap;
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index 60dd9c19b..b7b974c68 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -146,8 +146,6 @@ __glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
                                                 GF_DEFRAG_STATUS_STARTED) {
                                 volinfo->rebal.defrag_status =
                                                    GF_DEFRAG_STATUS_FAILED;
-                        } else {
-                                volinfo->rebal.defrag_cmd = 0;
                         }
                  }
 
@@ -219,6 +217,7 @@ glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
 
         defrag->cmd = cmd;
 
+        volinfo->rebal.defrag_cmd = cmd;
         volinfo->rebal.op = op;
 
         LOCK_INIT (&defrag->lock);
@@ -533,7 +532,7 @@ glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr)
         case GF_DEFRAG_CMD_START:
         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
         case GF_DEFRAG_CMD_START_FORCE:
-                if (is_origin_glusterd ()) {
+                if (is_origin_glusterd (dict)) {
                         op_ctx = glusterd_op_get_ctx ();
                         if (!op_ctx) {
                                 ret = -1;
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index 5c879b64c..54b830870 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -31,6 +31,7 @@
                   DEFAULT_VAR_RUN_DIRECTORY"/%s-"RB_CLIENT_MOUNTPOINT,      \
                   volinfo->volname);
 
+extern uuid_t global_txn_id;
 
 int
 glusterd_get_replace_op_str (gf1_cli_replace_op op, char *op_str)
@@ -268,13 +269,6 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                 goto out;
         }
 
-        if (volinfo->backend == GD_VOL_BK_BD) {
-                snprintf (msg, sizeof (msg), "replace brick not supported "
-                       "for Block backend volume");
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
         if (GLUSTERD_STATUS_STARTED != volinfo->status) {
                 ret = -1;
                 snprintf (msg, sizeof (msg), "volume: %s is not started",
@@ -332,7 +326,7 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
                         ret = -1;
                         goto out;
                 }
-                if (is_origin_glusterd ()) {
+                if (is_origin_glusterd (dict)) {
                         if (!ctx) {
                                 ret = -1;
                                 gf_log (this->name, GF_LOG_ERROR,
@@ -694,7 +688,7 @@ rb_src_brick_restart (glusterd_volinfo_t *volinfo,
 
         sleep (2);
         ret = glusterd_volume_start_glusterfs (volinfo, src_brickinfo,
-                                              _gf_false);
+                                               _gf_false);
         if (ret) {
                 gf_log ("", GF_LOG_ERROR, "Unable to start "
                         "glusterfs, ret: %d", ret);
@@ -1638,7 +1632,7 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
                 /* Set task-id, if available, in op_ctx dict for operations
                  * other than start
                  */
-                if  (is_origin_glusterd ()) {
+                if  (is_origin_glusterd (dict)) {
                         ctx = glusterd_op_get_ctx();
                         if (!ctx) {
                                 gf_log (this->name, GF_LOG_ERROR, "Failed to "
@@ -1901,6 +1895,7 @@ glusterd_do_replace_brick (void *data)
         glusterd_brickinfo_t   *src_brickinfo = NULL;
         glusterd_brickinfo_t   *dst_brickinfo = NULL;
 	glusterd_conf_t	       *priv = NULL;
+        uuid_t                 *txn_id = &global_txn_id;
 
         int ret = 0;
 
@@ -1920,6 +1915,10 @@ glusterd_do_replace_brick (void *data)
         gf_log ("", GF_LOG_DEBUG,
                 "Replace brick operation detected");
 
+        ret = dict_get_bin (dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
         ret = dict_get_int32 (dict, "operation", &op);
         if (ret) {
                 gf_log ("", GF_LOG_DEBUG,
@@ -2015,9 +2014,11 @@ glusterd_do_replace_brick (void *data)
 
 out:
         if (ret)
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, NULL);
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                                   txn_id, NULL);
         else
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_ACC, NULL);
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_ACC,
+                                                   txn_id, NULL);
 
         glusterd_op_sm ();
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
index 2d8d381bf..d5200a4ae 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
@@ -33,6 +33,7 @@
 
 
 extern glusterd_op_info_t opinfo;
+extern uuid_t global_txn_id;
 
 int32_t
 glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
@@ -142,7 +143,7 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
         case GD_OP_LIST_VOLUME:
         case GD_OP_CLEARLOCKS_VOLUME:
         case GD_OP_HEAL_VOLUME:
-        case GD_OP_BD_OP:
+        case GD_OP_SNAP:
         {
                 /*nothing specific to be done*/
                 break;
@@ -581,6 +582,7 @@ __glusterd_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
         glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
         glusterd_peerinfo_t           *peerinfo = NULL;
         xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = &global_txn_id;
 
         this = THIS;
         GF_ASSERT (this);
@@ -625,7 +627,7 @@ out:
                 event_type = GD_OP_EVENT_RCVD_ACC;
         }
 
-        ret = glusterd_op_sm_inject_event (event_type, NULL);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
         if (!ret) {
                 glusterd_friend_sm ();
@@ -644,6 +646,164 @@ glusterd_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
                                         __glusterd_cluster_lock_cbk);
 }
 
+static int32_t
+glusterd_mgmt_v3_lock_peers_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                                    int count, void *myframe)
+{
+        gd1_mgmt_v3_lock_rsp          rsp   = {{0},};
+        int                           ret   = -1;
+        int32_t                       op_ret = -1;
+        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
+        glusterd_peerinfo_t           *peerinfo = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to decode mgmt_v3 lock "
+                        "response received from peer");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+out:
+        op_ret = rsp.op_ret;
+
+        txn_id = &rsp.txn_id;
+
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received mgmt_v3 lock %s from uuid: %s",
+                (op_ret) ? "RJT" : "ACC", uuid_utoa (rsp.uuid));
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "mgmt_v3 lock response received "
+                        "from unknown peer: %s", uuid_utoa (rsp.uuid));
+        }
+
+        if (op_ret) {
+                event_type = GD_OP_EVENT_RCVD_RJT;
+                opinfo.op_ret = op_ret;
+                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
+                                              "progress. Please try again after"
+                                              " sometime.");
+        } else {
+                event_type = GD_OP_EVENT_RCVD_ACC;
+        }
+
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
+
+        if (!ret) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
+        }
+
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_lock_peers_cbk (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        glusterd_mgmt_v3_lock_peers_cbk_fn);
+}
+
+static int32_t
+glusterd_mgmt_v3_unlock_peers_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                                      int count, void *myframe)
+{
+        gd1_mgmt_v3_unlock_rsp        rsp   = {{0},};
+        int                           ret   = -1;
+        int32_t                       op_ret = -1;
+        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
+        glusterd_peerinfo_t           *peerinfo = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to decode mgmt_v3 unlock "
+                        "response received from peer");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+out:
+        op_ret = rsp.op_ret;
+
+        txn_id = &rsp.txn_id;
+
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received mgmt_v3 unlock %s from uuid: %s",
+                (op_ret) ? "RJT" : "ACC",
+                uuid_utoa (rsp.uuid));
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "mgmt_v3 unlock response received "
+                        "from unknown peer: %s", uuid_utoa (rsp.uuid));
+        }
+
+        if (op_ret) {
+                event_type = GD_OP_EVENT_RCVD_RJT;
+                opinfo.op_ret = op_ret;
+                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
+                                              "progress. Please try again after"
+                                              " sometime.");
+        } else {
+                event_type = GD_OP_EVENT_RCVD_ACC;
+        }
+
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
+
+        if (!ret) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
+        }
+
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_unlock_peers_cbk (struct rpc_req *req, struct iovec *iov,
+                                   int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        glusterd_mgmt_v3_unlock_peers_cbk_fn);
+}
+
 int32_t
 __glusterd_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
                                  int count, void *myframe)
@@ -654,6 +814,7 @@ __glusterd_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
         glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
         glusterd_peerinfo_t           *peerinfo = NULL;
         xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = &global_txn_id;
 
         this = THIS;
         GF_ASSERT (this);
@@ -695,7 +856,7 @@ out:
                 event_type = GD_OP_EVENT_RCVD_ACC;
         }
 
-        ret = glusterd_op_sm_inject_event (event_type, NULL);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
         if (!ret) {
                 glusterd_friend_sm ();
@@ -727,6 +888,7 @@ __glusterd_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
         char                          err_str[2048] = {0};
         char                          *peer_str = NULL;
         xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -779,6 +941,10 @@ out:
                 "Received stage %s from uuid: %s",
                 (op_ret) ? "RJT" : "ACC", uuid_utoa (rsp.uuid));
 
+        ret = dict_get_bin (dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
         ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
 
         if (ret) {
@@ -814,7 +980,7 @@ out:
                 break;
         }
 
-        ret = glusterd_op_sm_inject_event (event_type, NULL);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
         if (!ret) {
                 glusterd_friend_sm ();
@@ -854,6 +1020,8 @@ __glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
         char                          err_str[2048] = {0};
         char                          *peer_str = NULL;
         xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
+
 
         this = THIS;
         GF_ASSERT (this);
@@ -907,6 +1075,10 @@ __glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
                 "Received commit %s from uuid: %s",
                 (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
 
+        ret = dict_get_bin (dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
         ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
 
         if (ret) {
@@ -986,7 +1158,7 @@ __glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
         }
 
 out:
-        ret = glusterd_op_sm_inject_event (event_type, NULL);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
 
         if (!ret) {
                 glusterd_friend_sm ();
@@ -1223,6 +1395,134 @@ out:
 }
 
 int32_t
+glusterd_mgmt_v3_lock_peers (call_frame_t *frame, xlator_t *this,
+                             void *data)
+{
+        gd1_mgmt_v3_lock_req             req         = {{0},};
+        int                              ret         = -1;
+        glusterd_peerinfo_t             *peerinfo    = NULL;
+        glusterd_conf_t                 *priv        = NULL;
+        call_frame_t                    *dummy_frame = NULL;
+        dict_t                          *dict        = NULL;
+        uuid_t                          *txn_id      = NULL;
+
+        if (!this)
+                goto out;
+
+        dict = data;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
+        if (ret)
+                goto out;
+
+        //peerinfo should not be in payload
+        dict_del (dict, "peerinfo");
+
+        glusterd_get_uuid (&req.uuid);
+
+        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict "
+                        "to request buffer");
+                goto out;
+        }
+
+        /* Sending valid transaction ID to peers */
+        ret = dict_get_bin (dict, "transaction_id",
+                            (void **)&txn_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to get transaction id.");
+                goto out;
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Transaction_id = %s", uuid_utoa (*txn_id));
+                uuid_copy (req.txn_id, *txn_id);
+        }
+
+        dummy_frame = create_frame (this, this->ctx->pool);
+        if (!dummy_frame)
+                goto out;
+
+        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
+                                       peerinfo->mgmt_v3,
+                                       GLUSTERD_MGMT_V3_LOCK, NULL,
+                                       this, glusterd_mgmt_v3_lock_peers_cbk,
+                                       (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_unlock_peers (call_frame_t *frame, xlator_t *this,
+                               void *data)
+{
+        gd1_mgmt_v3_unlock_req           req         = {{0},};
+        int                              ret         = -1;
+        glusterd_peerinfo_t             *peerinfo    = NULL;
+        glusterd_conf_t                 *priv        = NULL;
+        call_frame_t                    *dummy_frame = NULL;
+        dict_t                          *dict        = NULL;
+        uuid_t                          *txn_id      = NULL;
+
+        if (!this)
+                goto out;
+
+        dict = data;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
+        if (ret)
+                goto out;
+
+        //peerinfo should not be in payload
+        dict_del (dict, "peerinfo");
+
+        glusterd_get_uuid (&req.uuid);
+
+        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict "
+                        "to request buffer");
+                goto out;
+        }
+
+        /* Sending valid transaction ID to peers */
+        ret = dict_get_bin (dict, "transaction_id",
+                            (void **)&txn_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to get transaction id.");
+                goto out;
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Transaction_id = %s", uuid_utoa (*txn_id));
+                uuid_copy (req.txn_id, *txn_id);
+        }
+
+        dummy_frame = create_frame (this, this->ctx->pool);
+        if (!dummy_frame)
+                goto out;
+
+        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
+                                       peerinfo->mgmt_v3,
+                                       GLUSTERD_MGMT_V3_UNLOCK, NULL,
+                                       this, glusterd_mgmt_v3_unlock_peers_cbk,
+                                       (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
 glusterd_cluster_unlock (call_frame_t *frame, xlator_t *this,
                             void *data)
 {
@@ -1385,6 +1685,7 @@ __glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
         glusterd_req_ctx_t            *req_ctx = NULL;
         glusterd_pending_node_t       *node = NULL;
         xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = &global_txn_id;
 
         this = THIS;
         GF_ASSERT (this);
@@ -1447,6 +1748,11 @@ __glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
                 }
         }
 out:
+
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
         ev_ctx = GF_CALLOC (1, sizeof (*ev_ctx), gf_gld_mt_brick_rsp_ctx_t);
         GF_ASSERT (ev_ctx);
         if (op_ret) {
@@ -1459,7 +1765,7 @@ out:
         ev_ctx->pending_node = frame->cookie;
         ev_ctx->rsp_dict  = dict;
         ev_ctx->commit_ctx = frame->local;
-        ret = glusterd_op_sm_inject_event (event_type, ev_ctx);
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, ev_ctx);
         if (!ret) {
                 glusterd_friend_sm ();
                 glusterd_op_sm ();
@@ -1484,6 +1790,7 @@ int32_t
 glusterd_brick_op (call_frame_t *frame, xlator_t *this,
                       void *data)
 {
+
         gd1_mgmt_brick_op_req           *req = NULL;
         int                             ret = 0;
         glusterd_conf_t                 *priv = NULL;
@@ -1494,6 +1801,7 @@ glusterd_brick_op (call_frame_t *frame, xlator_t *this,
         glusterd_req_ctx_t              *req_ctx = NULL;
         struct rpc_clnt                 *rpc = NULL;
         dict_t                          *op_ctx = NULL;
+        uuid_t                          *txn_id = &global_txn_id;
 
         if (!this) {
                 ret = -1;
@@ -1516,6 +1824,11 @@ glusterd_brick_op (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
+
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
         list_for_each_entry (pending_node, &opinfo.pending_bricks, list) {
                 dummy_frame = create_frame (this, this->ctx->pool);
                 if (!dummy_frame)
@@ -1593,7 +1906,8 @@ glusterd_brick_op (call_frame_t *frame, xlator_t *this,
 
 out:
         if (ret) {
-                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, data);
+                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                             txn_id, data);
                 opinfo.op_ret = ret;
         }
         gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
@@ -1621,6 +1935,12 @@ struct rpc_clnt_procedure gd_mgmt_actors[GLUSTERD_MGMT_MAXVALUE] = {
         [GLUSTERD_MGMT_COMMIT_OP]      = {"COMMIT_OP", glusterd_commit_op},
 };
 
+struct rpc_clnt_procedure gd_mgmt_v3_actors[GLUSTERD_MGMT_V3_MAXVALUE] = {
+        [GLUSTERD_MGMT_V3_NULL]           = {"NULL", NULL },
+        [GLUSTERD_MGMT_V3_LOCK]           = {"MGMT_V3_LOCK", glusterd_mgmt_v3_lock_peers},
+        [GLUSTERD_MGMT_V3_UNLOCK]         = {"MGMT_V3_UNLOCK", glusterd_mgmt_v3_unlock_peers},
+};
+
 struct rpc_clnt_program gd_mgmt_prog = {
         .progname  = "glusterd mgmt",
         .prognum   = GD_MGMT_PROGRAM,
@@ -1645,4 +1965,10 @@ struct rpc_clnt_program gd_peer_prog = {
         .numproc   = GLUSTERD_FRIEND_MAXVALUE,
 };
 
-
+struct rpc_clnt_program gd_mgmt_v3_prog = {
+        .progname  = "glusterd mgmt v3",
+        .prognum   = GD_MGMT_V3_PROGRAM,
+        .progver   = GD_MGMT_V3_VERSION,
+        .proctable = gd_mgmt_v3_actors,
+        .numproc   = GLUSTERD_MGMT_V3_MAXVALUE,
+};
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.h b/xlators/mgmt/glusterd/src/glusterd-sm.h
index e29bb7277..b9bedbe69 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.h
@@ -97,6 +97,7 @@ struct glusterd_peerinfo_ {
         struct rpc_clnt                 *rpc;
         rpc_clnt_prog_t                 *mgmt;
         rpc_clnt_prog_t                 *peer;
+        rpc_clnt_prog_t                 *mgmt_v3;
         int                             connected;
         gf_store_handle_t         *shandle;
         glusterd_sm_tr_log_t            sm_log;
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot.c b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
new file mode 100644
index 000000000..9b811cd05
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
@@ -0,0 +1,5590 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <inttypes.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/resource.h>
+#include <sys/statvfs.h>
+#include <sys/mount.h>
+
+#include "globals.h"
+#include "compat.h"
+#include "protocol-common.h"
+#include "xlator.h"
+#include "logging.h"
+#include "timer.h"
+#include "glusterd-mem-types.h"
+#include "glusterd.h"
+#include "glusterd-sm.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-utils.h"
+#include "glusterd-store.h"
+#include "run.h"
+#include "glusterd-volgen.h"
+#include "glusterd-mgmt.h"
+#include "glusterd-syncop.h"
+
+#include "syscall.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+
+#ifdef GF_LINUX_HOST_OS
+#include <mntent.h>
+#endif
+
+char snap_mount_folder[PATH_MAX];
+
+static int32_t
+glusterd_find_missed_snap (dict_t *rsp_dict, glusterd_volinfo_t *vol,
+                           char *snap_uuid, struct list_head *peers,
+                           int32_t op);
+
+/* This function will restore a snapshot volumes
+ *
+ * @param dict          dictionary containing snapshot restore request
+ * @param op_errstr     In case of any failure error message will be returned
+ *                      in this variable
+ * @return              Negative value on Failure and 0 in success
+ */
+int
+glusterd_snapshot_restore (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        int                     ret             = -1;
+        char                    *volname        = NULL;
+        char                    *snapname       = NULL;
+        xlator_t                *this           = NULL;
+        glusterd_volinfo_t      *snap_volinfo   = NULL;
+        glusterd_volinfo_t      *volinfo        = NULL;
+        glusterd_snap_t         *snap           = NULL;
+        glusterd_conf_t         *priv           = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                        "snap name");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (NULL == snap) {
+                ret = gf_asprintf (op_errstr, "Snap (%s) not found",
+                                        snapname);
+                if (ret < 0) {
+                        goto out;
+                }
+                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                ret = -1;
+                goto out;
+        }
+
+        /* TODO : As of now there is only volume in snapshot.
+         * Change this when multiple volume snapshot is introduced
+         */
+        snap_volinfo = list_entry (snap->volumes.next, glusterd_volinfo_t,
+                                   vol_list);
+
+        ret = glusterd_volinfo_find (snap_volinfo->parent_volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not get volinfo of "
+                        "%s", snap_volinfo->parent_volname);
+                goto out;
+        }
+
+        if (is_origin_glusterd (dict) == _gf_true) {
+                /* From origin glusterd check if      *
+                 * any peers with snap bricks is down */
+                ret = glusterd_find_missed_snap (rsp_dict, snap_volinfo,
+                                                 snap_volinfo->volname,
+                                                 &priv->peers,
+                                                 GF_SNAP_OPTION_TYPE_RESTORE);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to find missed snap restores");
+                        goto out;
+                }
+        }
+
+        ret = gd_restore_snap_volume (rsp_dict, volinfo, snap_volinfo);
+        if (ret) {
+                /* No need to update op_errstr because it is assumed
+                 * that the called function will do that in case of
+                 * failure.
+                 */
+                gf_log (this->name, GF_LOG_ERROR, "Failed to restore "
+                        "snap for %s volume", volname);
+                goto out;
+        }
+
+        ret = 0;
+
+        /* TODO: Need to check if we need to delete the snap after the
+         * operation is successful or not. Also need to persist the state
+         * of restore operation in the store.
+         */
+out:
+        return ret;
+}
+
+/* This function is called before actual restore is taken place. This function
+ * will validate whether the snapshot volumes are ready to be restored or not.
+ *
+ * @param dict          dictionary containing snapshot restore request
+ * @param op_errstr     In case of any failure error message will be returned
+ *                      in this variable
+ * @param rsp_dict      response dictionary
+ * @return              Negative value on Failure and 0 in success
+ */
+int
+glusterd_snapshot_restore_prevalidate (dict_t *dict, char **op_errstr,
+                                       dict_t *rsp_dict)
+{
+        int                     ret             = -1;
+        int32_t                 i               = 0;
+        int32_t                 volcount        = 0;
+        gf_boolean_t            snap_restored   = _gf_false;
+        char                    key[PATH_MAX]   = {0, };
+        char                    *volname        = NULL;
+        char                    *snapname       = NULL;
+        glusterd_volinfo_t      *volinfo        = NULL;
+        glusterd_snap_t         *snap           = NULL;
+        xlator_t                *this           = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                        "snap name");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (NULL == snap) {
+                ret = gf_asprintf (op_errstr, "Snap (%s) not found",
+                                snapname);
+                if (ret < 0) {
+                        goto out;
+                }
+                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                ret = -1;
+                goto out;
+        }
+
+        snap_restored = snap->snap_restored;
+
+        if (snap_restored) {
+                ret = gf_asprintf (op_errstr, "Snap (%s) is already "
+                                  "restored", snapname);
+                if (ret < 0) {
+                        goto out;
+                }
+                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (rsp_dict, "snapname", snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                        "snap name");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "volcount", &volcount);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume count");
+                goto out;
+        }
+
+        /* Snapshot restore will only work if all the volumes,
+           that are part of the snapshot, are stopped. */
+        for (i = 1; i <= volcount; ++i) {
+                snprintf (key, sizeof (key), "volname%d", i);
+                ret = dict_get_str (dict, key, &volname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "get volume name");
+                        goto out;
+                }
+
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        ret = gf_asprintf (op_errstr, "Volume (%s) not found",
+                                           volname);
+                        if (ret < 0) {
+                                goto out;
+                        }
+                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                        ret = -1;
+                        goto out;
+                }
+
+                if (glusterd_is_volume_started (volinfo)) {
+                        ret = gf_asprintf (op_errstr, "Volume (%s) has been "
+                        "started. Volume needs to be stopped before restoring "
+                        "a snapshot.", volname);
+                        if (ret < 0) {
+                                goto out;
+                        }
+                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+snap_max_hard_limits_validate (dict_t *dict, char *volname,
+                          uint64_t value, char **op_errstr)
+{
+        char                err_str[PATH_MAX] = "";
+        glusterd_conf_t    *conf              = NULL;
+        glusterd_volinfo_t *volinfo           = NULL;
+        int                 ret               = -1;
+        uint64_t            max_limit         = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+        xlator_t           *this              = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        if (volname) {
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (!ret) {
+                        if (volinfo->is_snap_volume) {
+                                ret = -1;
+                                snprintf (err_str, PATH_MAX,
+                                         "%s is a snap volume. Configuring "
+                                         "snap-max-hard-limit for a snap "
+                                         "volume is prohibited.", volname);
+                                goto out;
+                        }
+                }
+        }
+
+        if (value) {
+                /* Max limit for the system is GLUSTERD_SNAPS_MAX_HARD_LIMIT
+                 * but max limit for a volume is conf->snap_max_hard_limit.
+                 */
+                if (volname) {
+                        max_limit = conf->snap_max_hard_limit;
+                } else {
+                        max_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+                }
+        }
+
+        if ((value < 0) || (value > max_limit)) {
+                ret = -1;
+                snprintf (err_str, PATH_MAX, "Invalid snap-max-hard-limit"
+                          "%"PRIu64 ". Expected range 0 - %"PRIu64,
+                          value, max_limit);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (ret) {
+                *op_errstr = gf_strdup (err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+        }
+        return ret;
+}
+
+int
+glusterd_snapshot_config_prevalidate (dict_t *dict, char **op_errstr)
+{
+        char               *volname             = NULL;
+        glusterd_volinfo_t *volinfo             = NULL;
+        xlator_t           *this                = NULL;
+        int                 ret                 = -1;
+        int                 config_command      = 0;
+        char                err_str[PATH_MAX]   = {0,};
+        glusterd_conf_t    *conf                = NULL;
+        uint64_t            value               = 0;
+        uint64_t            hard_limit          = 0;
+        uint64_t            soft_limit          = 0;
+        gf_loglevel_t       loglevel            = GF_LOG_ERROR;
+        uint64_t            max_limit           = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        ret = dict_get_int32 (dict, "config-command", &config_command);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str),
+                          "failed to get config-command type");
+                goto out;
+        }
+
+        ret = dict_get_uint64 (dict, "snap-max-hard-limit", &hard_limit);
+
+        ret = dict_get_uint64 (dict, "snap-max-soft-limit", &soft_limit);
+
+        ret = dict_get_str (dict, "volname", &volname);
+
+        if (volname) {
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "Volume %s does not exist.", volname);
+                        goto out;
+                }
+        }
+
+        switch (config_command) {
+        case GF_SNAP_CONFIG_TYPE_SET:
+                if (hard_limit) {
+                        /* Validations for snap-max-hard-limits */
+                        ret = snap_max_hard_limits_validate (dict, volname,
+                                                         hard_limit, op_errstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "snap-max-hard-limit validation "
+                                        "failed.");
+                                goto out;
+                        }
+                }
+
+                if (soft_limit) {
+                        max_limit = GLUSTERD_SNAPS_MAX_SOFT_LIMIT_PERCENT;
+                        if ((soft_limit < 0) || (soft_limit > max_limit)) {
+                                ret = -1;
+                                snprintf (err_str, PATH_MAX, "Invalid "
+                                         "snap-max-soft-limit ""%"
+                                         PRIu64 ". Expected range 0 - %"PRIu64,
+                                         value, max_limit);
+                                goto out;
+                        }
+                        break;
+                }
+        default:
+                break;
+        }
+
+        ret = 0;
+out:
+
+        if (ret && err_str[0] != '\0') {
+                gf_log (this->name, loglevel, "%s", err_str);
+                *op_errstr = gf_strdup (err_str);
+        }
+
+        return ret;
+}
+
+int
+glusterd_snap_create_pre_val_use_rsp_dict (dict_t *dst, dict_t *src)
+{
+        char                  *snap_brick_dir        = NULL;
+        char                  *snap_device           = NULL;
+        char                  *tmpstr                = NULL;
+        char                   key[PATH_MAX]         = "";
+        char                   snapbrckcnt[PATH_MAX] = "";
+        char                   snapbrckord[PATH_MAX] = "";
+        int                    ret                   = -1;
+        int64_t                i                     = -1;
+        int64_t                j                     = -1;
+        int64_t                volume_count          = 0;
+        int64_t                brick_count           = 0;
+        int64_t                brick_order           = 0;
+        xlator_t              *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dst);
+        GF_ASSERT (src);
+
+        ret = dict_get_int64 (src, "volcount", &volume_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                        "get the volume count");
+                goto out;
+        }
+
+        for (i = 0; i < volume_count; i++) {
+                memset (snapbrckcnt, '\0', sizeof(snapbrckcnt));
+                ret = snprintf (snapbrckcnt, sizeof(snapbrckcnt) - 1,
+                                "vol%ld_brickcount", i+1);
+                ret = dict_get_int64 (src, snapbrckcnt, &brick_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "No bricks for this volume in this dict");
+                        continue;
+                }
+
+                for (j = 0; j < brick_count; j++) {
+                        /* Fetching data from source dict */
+                        snprintf (key, sizeof(key) - 1,
+                                  "vol%ld.brickdir%ld", i+1, j);
+
+                        ret = dict_get_ptr (src, key,
+                                            (void **)&snap_brick_dir);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Unable to fetch %s", key);
+                                continue;
+                        }
+
+                        snprintf (key, sizeof(key) - 1,
+                                  "vol%ld.brick_snapdevice%ld", i+1, j);
+
+                        ret = dict_get_ptr (src, key,
+                                            (void **)&snap_device);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to fetch snap_device");
+                                goto out;
+                        }
+
+                        snprintf (snapbrckord, sizeof(snapbrckord) - 1,
+                                  "vol%ld.brick%ld.order", i+1, j);
+
+                        ret = dict_get_int64 (src, snapbrckord, &brick_order);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get brick order");
+                                goto out;
+                        }
+
+                        /* Adding the data in the dst dict */
+                        snprintf (key, sizeof(key) - 1,
+                                  "vol%ld.brickdir%ld", i+1, brick_order);
+
+                        tmpstr = gf_strdup (snap_brick_dir);
+                        if (!tmpstr) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Out Of Memory");
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = dict_set_dynstr (dst, key, tmpstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", key);
+                                GF_FREE (tmpstr);
+                                goto out;
+                        }
+
+                        snprintf (key, sizeof(key) - 1,
+                                  "vol%ld.brick_snapdevice%ld",
+                                  i+1, brick_order);
+
+                        tmpstr = gf_strdup (snap_device);
+                        if (!tmpstr) {
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = dict_set_dynstr (dst, key, tmpstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", key);
+                                GF_FREE (tmpstr);
+                                goto out;
+                        }
+
+                }
+        }
+
+        ret = 0;
+out:
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_snap_pre_validate_use_rsp_dict (dict_t *dst, dict_t *src)
+{
+        int             ret             = -1;
+        int32_t         snap_command    = 0;
+        xlator_t       *this            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (!dst || !src) {
+                gf_log (this->name, GF_LOG_ERROR, "Source or Destination "
+                        "dict is empty.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dst, "type", &snap_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+                ret = glusterd_snap_create_pre_val_use_rsp_dict (dst, src);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to use "
+                                "rsp dict");
+                        goto out;
+                }
+                break;
+        default:
+                break;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_snapshot_create_prevalidate (dict_t *dict, char **op_errstr,
+                                      dict_t *rsp_dict)
+{
+        char                  *volname                   = NULL;
+        char                  *snapname                  = NULL;
+        char                  *device                    = NULL;
+        char                  *tmpstr                    = NULL;
+        char                  *brick_dir                 = NULL;
+        char                   snap_brick_dir[PATH_MAX]  = "";
+        char                  *mnt_pt                    = NULL;
+        char                   key[PATH_MAX]             = "";
+        char                   snap_mount[PATH_MAX]      = "";
+        char                   snap_volname[64]          = "";
+        char                   err_str[PATH_MAX]         = "";
+        int                    ret                       = -1;
+        int64_t                i                         = 0;
+        int64_t                volcount                  = 0;
+        int64_t                brick_count               = 0;
+        int64_t                brick_order               = 0;
+        glusterd_brickinfo_t  *brickinfo                 = NULL;
+        glusterd_volinfo_t    *volinfo                   = NULL;
+        xlator_t              *this                      = NULL;
+        uuid_t                *snap_volid                = NULL;
+        gf_loglevel_t          loglevel                  = GF_LOG_ERROR;
+        glusterd_conf_t       *conf                      = NULL;
+        int64_t                effective_max_limit       = 0;
+
+        this = THIS;
+        GF_ASSERT (op_errstr);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        ret = dict_get_int64 (dict, "volcount", &volcount);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Failed to "
+                          "get the volume count");
+                goto out;
+        }
+        if (volcount <= 0) {
+                snprintf (err_str, sizeof (err_str), "Invalid volume count %ld "
+                          "supplied", volcount);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Failed to get snapname");
+                goto out;
+        }
+
+        if (glusterd_find_snap_by_name (snapname)) {
+                ret = -1;
+                snprintf (err_str, sizeof (err_str), "Snap %s already exists",
+                          snapname);
+                goto out;
+        }
+
+        for (i = 1; i <= volcount; i++) {
+                snprintf (key, sizeof (key), "volname%ld", i);
+                ret = dict_get_str (dict, key, &volname);
+                if (ret) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "failed to get volume name");
+                        goto out;
+                }
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "Volume (%s) does not exist ", volname);
+                        goto out;
+                }
+
+                ret = -1;
+                if (!glusterd_is_volume_started (volinfo)) {
+                        snprintf (err_str, sizeof (err_str), "volume %s is "
+                                  "not started", volinfo->volname);
+                        loglevel = GF_LOG_WARNING;
+                        goto out;
+                }
+                if (glusterd_is_defrag_on (volinfo)) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "rebalance process is running for the "
+                                  "volume %s", volname);
+                        loglevel = GF_LOG_WARNING;
+                        goto out;
+                }
+                /* TODO: Also check whether geo replication is running */
+
+                if (volinfo->is_snap_volume == _gf_true) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "Volume %s is a snap volume", volname);
+                        loglevel = GF_LOG_WARNING;
+                        goto out;
+                }
+
+                if (volinfo->snap_max_hard_limit < conf->snap_max_hard_limit)
+                        effective_max_limit = volinfo->snap_max_hard_limit;
+                else
+                        effective_max_limit = conf->snap_max_hard_limit;
+
+                if (volinfo->snap_count >= effective_max_limit) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "The number of existing snaps has reached "
+                                  "the effective maximum limit of %"PRIu64" ,"
+                                  "for the volume %s", effective_max_limit,
+                                  volname);
+                        loglevel = GF_LOG_WARNING;
+                        goto out;
+                }
+
+                snprintf (key, sizeof(key) - 1, "vol%ld_volid", i);
+                ret = dict_get_bin (dict, key, (void **)&snap_volid);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to fetch snap_volid");
+                        goto out;
+                }
+
+                /* snap volume uuid is used as lvm snapshot name.
+                   This will avoid restrictions on snapshot names
+                   provided by user */
+                GLUSTERD_GET_UUID_NOHYPHEN (snap_volname, *snap_volid);
+
+                brick_count = 0;
+                brick_order = 0;
+                /* Adding snap bricks mount paths to the dict */
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        if (uuid_compare (brickinfo->uuid, MY_UUID)) {
+                                brick_order++;
+                                continue;
+                        }
+
+                        if (!glusterd_is_brick_started (brickinfo)) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "brick %s:%s is not started",
+                                        brickinfo->hostname,
+                                        brickinfo->path);
+                                brick_order++;
+                                brick_count++;
+                                continue;
+                        }
+
+                        device = glusterd_get_brick_mount_details (brickinfo);
+                        if (!device) {
+                                snprintf (err_str, sizeof (err_str),
+                                          "getting device name for the brick "
+                                          "%s:%s failed", brickinfo->hostname,
+                                          brickinfo->path);
+                                ret = -1;
+                                goto out;
+                        }
+
+                        device = glusterd_build_snap_device_path (device,
+                                                                  snap_volname);
+                        if (!device) {
+                                snprintf (err_str, sizeof (err_str),
+                                          "cannot copy the snapshot device "
+                                          "name (volname: %s, snapname: %s)",
+                                          volinfo->volname, snapname);
+                                loglevel = GF_LOG_WARNING;
+                                ret = -1;
+                                goto out;
+                        }
+
+                        snprintf (key, sizeof(key),
+                                  "vol%ld.brick_snapdevice%ld", i,
+                                  brick_count);
+                        ret = dict_set_dynstr (rsp_dict, key, device);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", key);
+                                GF_FREE (device);
+                                goto out;
+                        }
+
+                        ret = glusterd_get_brick_root (brickinfo->path,
+                                                       &mnt_pt);
+                        if (ret) {
+                                snprintf (err_str, sizeof (err_str),
+                                "could not get the root of the brick path %s",
+                                brickinfo->path);
+                                loglevel = GF_LOG_WARNING;
+                                goto out;
+                        }
+                        if (strncmp (brickinfo->path, mnt_pt, strlen(mnt_pt))) {
+                                snprintf (err_str, sizeof (err_str),
+                                "brick: %s brick mount: %s",
+                                 brickinfo->path, mnt_pt);
+                                loglevel = GF_LOG_WARNING;
+                                goto out;
+                        }
+
+                        brick_dir = &brickinfo->path[strlen (mnt_pt)];
+                        brick_dir++;
+
+                        snprintf (snap_brick_dir, sizeof (snap_brick_dir),
+                                  "/%s", brick_dir);
+
+                        tmpstr = gf_strdup (snap_brick_dir);
+                        if (!tmpstr) {
+                                ret = -1;
+                                goto out;
+                        }
+                        snprintf (key, sizeof(key), "vol%ld.brickdir%ld", i,
+                                  brick_count);
+                        ret = dict_set_dynstr (rsp_dict, key, tmpstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", snap_mount);
+                                goto out;
+                        }
+                        tmpstr = NULL;
+
+                        snprintf (key, sizeof(key) - 1, "vol%ld.brick%ld.order",
+                                  i, brick_count);
+                        ret = dict_set_int64 (rsp_dict, key, brick_order);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", key);
+                                goto out;
+                        }
+
+                        brick_count++;
+                        brick_order++;
+                }
+                snprintf (key, sizeof(key) - 1, "vol%ld_brickcount", i);
+                ret = dict_set_int64 (rsp_dict, key, brick_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set %s",
+                                key);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int64 (rsp_dict, "volcount", volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set volcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (ret)
+                GF_FREE (tmpstr);
+
+        if (ret && err_str[0] != '\0') {
+                gf_log (this->name, loglevel, "%s", err_str);
+                *op_errstr = gf_strdup (err_str);
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+glusterd_snap_t*
+glusterd_new_snap_object()
+{
+        glusterd_snap_t        *snap    = NULL;
+
+        snap = GF_CALLOC (1, sizeof (*snap), gf_gld_mt_snap_t);
+
+        if (snap) {
+                if (LOCK_INIT (&snap->lock)) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "Failed initiating"
+                                " snap lock");
+                        GF_FREE (snap);
+                        return NULL;
+                }
+
+                INIT_LIST_HEAD (&snap->snap_list);
+                INIT_LIST_HEAD (&snap->volumes);
+                snap->snapname[0] = 0;
+                snap->snap_status = GD_SNAP_STATUS_INIT;
+        }
+
+        return snap;
+
+};
+
+/* Function glusterd_list_add_snapvol adds the volinfo object (snapshot volume)
+   to the snapshot object list and to the parent volume list */
+int32_t
+glusterd_list_add_snapvol (glusterd_volinfo_t *origin_vol,
+                           glusterd_volinfo_t *snap_vol)
+{
+        int ret               = -1;
+        glusterd_snap_t *snap = NULL;
+
+        GF_VALIDATE_OR_GOTO ("glusterd", origin_vol, out);
+        GF_VALIDATE_OR_GOTO ("glusterd", snap_vol, out);
+
+        snap = snap_vol->snapshot;
+        GF_ASSERT (snap);
+
+        list_add_tail (&snap_vol->vol_list, &snap->volumes);
+        LOCK (&origin_vol->lock);
+        {
+                list_add_order (&snap_vol->snapvol_list,
+                               &origin_vol->snap_volumes,
+                               glusterd_compare_snap_vol_time);
+                origin_vol->snap_count++;
+        }
+        UNLOCK (&origin_vol->lock);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Snap %s added to the list",
+                snap->snapname);
+        ret = 0;
+out:
+        return ret;
+}
+
+glusterd_snap_t*
+glusterd_find_snap_by_name (char *snapname)
+{
+        glusterd_snap_t *snap  = NULL;
+        glusterd_conf_t *priv  = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snapname);
+
+
+        list_for_each_entry (snap, &priv->snapshots, snap_list) {
+                if (!strcmp (snap->snapname, snapname)) {
+                        gf_log (THIS->name, GF_LOG_DEBUG, "Found "
+                                "snap %s (%s)", snap->snapname,
+                                uuid_utoa (snap->snap_id));
+                        goto out;
+                }
+        }
+        snap = NULL;
+out:
+        return snap;
+}
+
+glusterd_snap_t*
+glusterd_find_snap_by_id (uuid_t snap_id)
+{
+        glusterd_snap_t *snap  = NULL;
+        glusterd_conf_t *priv  = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        if (uuid_is_null(snap_id))
+                goto out;
+
+        list_for_each_entry (snap, &priv->snapshots, snap_list) {
+                if (!uuid_compare (snap->snap_id, snap_id)) {
+                        gf_log (THIS->name, GF_LOG_DEBUG, "Found "
+                                "snap %s (%s)", snap->snapname,
+                                uuid_utoa (snap->snap_id));
+                        goto out;
+                }
+        }
+        snap = NULL;
+out:
+        return snap;
+}
+
+int
+glusterd_do_lvm_snapshot_remove (glusterd_volinfo_t *snap_vol,
+                                 glusterd_brickinfo_t *brickinfo,
+                                 const char *mount_pt, const char *snap_device)
+{
+        int                     ret               = -1;
+        xlator_t               *this              = NULL;
+        glusterd_conf_t        *priv              = NULL;
+        runner_t                runner            = {0,};
+        char                    msg[1024]         = {0, };
+        char                    pidfile[PATH_MAX] = {0, };
+        pid_t                   pid               = -1;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (!brickinfo) {
+                gf_log (this->name, GF_LOG_ERROR, "brickinfo NULL");
+                goto out;
+        }
+
+        GF_ASSERT (snap_vol);
+        GF_ASSERT (mount_pt);
+        GF_ASSERT (snap_device);
+
+        GLUSTERD_GET_BRICK_PIDFILE (pidfile, snap_vol, brickinfo, priv);
+        if (glusterd_is_service_running (pidfile, &pid)) {
+                ret = kill (pid, SIGKILL);
+                if (ret && errno != ESRCH) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to kill pid "
+                                "%d reason : %s", pid, strerror(errno));
+                        goto out;
+                }
+        }
+
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "umount the snapshot mounted path %s",
+                  mount_pt);
+        runner_add_args (&runner, "umount", mount_pt, NULL);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+
+        /* We need not do synclock_unlock => runner_run => synclock_lock here.
+           Because it is needed if we are running a glusterfs process in
+           runner_run, so that when the glusterfs process started wants to
+           communicate to glusterd, glusterd wont be able to respond if it
+           has held the big lock. So we do unlock, run glusterfs process
+           (thus communicate to glusterd), lock. But since this is not a
+           glusterfs command that is being run, unlocking and then relocking
+           is not needed.
+        */
+        ret = runner_run (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "unmounting the "
+                        "path %s (brick: %s) failed (%s)", mount_pt,
+                        brickinfo->path, strerror (errno));
+                goto out;
+        }
+
+        runinit (&runner);
+        snprintf (msg, sizeof(msg), "remove snapshot of the brick %s:%s, "
+                  "device: %s", brickinfo->hostname, brickinfo->path,
+                  snap_device);
+        runner_add_args (&runner, "/sbin/lvremove", "-f", snap_device, NULL);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+
+        ret = runner_run (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "removing snapshot of the "
+                        "brick (%s:%s) of device %s failed",
+                        brickinfo->hostname, brickinfo->path, snap_device);
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol)
+{
+        char                 *mnt_pt        = NULL;
+        struct mntent        *entry         = NULL;
+        int32_t               brick_count   = -1;
+        int32_t               ret           = -1;
+        glusterd_brickinfo_t *brickinfo     = NULL;
+        xlator_t             *this          = NULL;
+        FILE                 *mtab          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap_vol);
+
+        if (!snap_vol) {
+                gf_log (this->name, GF_LOG_ERROR, "snap volinfo is NULL");
+                goto out;
+        }
+
+        brick_count = -1;
+        list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
+                brick_count++;
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                if (brickinfo->snap_status == -1) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "snapshot was pending. lvm not present "
+                                "for brick %s:%s of the snap %s.",
+                                brickinfo->hostname, brickinfo->path,
+                                snap_vol->snapshot->snapname);
+
+                        /* Adding missed delete to the dict */
+                        ret = glusterd_add_missed_snaps_to_dict
+                                                   (rsp_dict,
+                                                    snap_vol->volname,
+                                                    brickinfo,
+                                                    brick_count + 1,
+                                                    GF_SNAP_OPTION_TYPE_DELETE);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to add missed snapshot info "
+                                        "for %s:%s in the rsp_dict",
+                                        brickinfo->hostname,
+                                        brickinfo->path);
+                                goto out;
+                        }
+
+                        continue;
+                }
+
+                ret = glusterd_get_brick_root (brickinfo->path, &mnt_pt);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "getting the root "
+                               "of the brick for volume %s (snap %s) failed ",
+                               snap_vol->volname, snap_vol->snapshot->snapname);
+                        goto out;
+                }
+
+                entry = glusterd_get_mnt_entry_info (mnt_pt, mtab);
+                if (!entry) {
+                        gf_log (this->name, GF_LOG_WARNING, "getting the mount"
+                                " entry for the brick %s:%s of the snap %s "
+                                "(volume: %s) failed", brickinfo->hostname,
+                                brickinfo->path, snap_vol->snapshot->snapname,
+                                snap_vol->volname);
+                        ret = -1;
+                        goto out;
+                }
+                ret = glusterd_do_lvm_snapshot_remove (snap_vol, brickinfo,
+                                                       mnt_pt,
+                                                       entry->mnt_fsname);
+                if (mtab)
+                        endmntent (mtab);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "remove the snapshot %s (%s)",
+                                brickinfo->path, entry->mnt_fsname);
+                        goto out;
+                }
+
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_snap_volume_remove (dict_t *rsp_dict,
+                             glusterd_volinfo_t *snap_vol,
+                             gf_boolean_t remove_lvm,
+                             gf_boolean_t force)
+{
+        int                   ret         = -1;
+        int                   save_ret    =  0;
+        glusterd_brickinfo_t *brickinfo   = NULL;
+        glusterd_volinfo_t   *origin_vol  = NULL;
+        xlator_t             *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap_vol);
+
+        if (!snap_vol) {
+                gf_log(this->name, GF_LOG_WARNING, "snap_vol in NULL");
+                ret = -1;
+                goto out;
+        }
+
+        list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                ret = glusterd_brick_stop (snap_vol, brickinfo, _gf_false);
+                if (ret) {
+                        gf_log(this->name, GF_LOG_WARNING, "Failed to stop "
+                               "brick for volume %s", snap_vol->volname);
+                        save_ret = ret;
+
+                        /* Continue to cleaning up the snap in case of error
+                           if force flag is enabled */
+                        if (!force)
+                                goto out;
+                }
+        }
+
+        /* Only remove the backend lvm when required */
+        if (remove_lvm) {
+                ret = glusterd_lvm_snapshot_remove (rsp_dict, snap_vol);
+                if (ret) {
+                        gf_log(this->name, GF_LOG_WARNING, "Failed to remove "
+                               "lvm snapshot volume %s", snap_vol->volname);
+                        save_ret = ret;
+                        if (!force)
+                                goto out;
+                }
+        }
+
+        ret = glusterd_store_delete_volume (snap_vol);
+        if (ret) {
+                gf_log(this->name, GF_LOG_WARNING, "Failed to remove volume %s "
+                       "from store", snap_vol->volname);
+                save_ret = ret;
+                if (!force)
+                        goto out;
+        }
+
+        if (!list_empty(&snap_vol->snapvol_list)) {
+                ret = glusterd_volinfo_find (snap_vol->parent_volname,
+                                             &origin_vol);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "parent volinfo %s  for volume  %s",
+                                snap_vol->parent_volname, snap_vol->volname);
+                        save_ret = ret;
+                        if (!force)
+                                goto out;
+                }
+                origin_vol->snap_count--;
+        }
+
+        ret = glusterd_volinfo_delete (snap_vol);
+        if (ret) {
+                gf_log(this->name, GF_LOG_WARNING, "Failed to remove volinfo "
+                       "%s ", snap_vol->volname);
+                save_ret = ret;
+                if (!force)
+                        goto out;
+        }
+
+        if (save_ret)
+                ret = save_ret;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_snapobject_delete (glusterd_snap_t *snap)
+{
+        if (snap == NULL) {
+                gf_log(THIS->name, GF_LOG_WARNING, "snap is NULL");
+                return -1;
+        }
+
+        list_del_init (&snap->snap_list);
+        list_del_init (&snap->volumes);
+        if (LOCK_DESTROY(&snap->lock))
+                gf_log (THIS->name, GF_LOG_WARNING, "Failed destroying lock"
+                        "of snap %s", snap->snapname);
+
+        GF_FREE (snap->description);
+        GF_FREE (snap);
+
+        return 0;
+}
+
+int32_t
+glusterd_snap_remove (dict_t *rsp_dict,
+                      glusterd_snap_t *snap,
+                      gf_boolean_t remove_lvm,
+                      gf_boolean_t force)
+{
+        int                 ret       = -1;
+        int                 save_ret  =  0;
+        glusterd_volinfo_t *snap_vol  = NULL;
+        glusterd_volinfo_t *tmp       = NULL;
+        xlator_t           *this      = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap);
+
+        if (!snap) {
+                gf_log(this->name, GF_LOG_WARNING, "snap is NULL");
+                ret = -1;
+                goto out;
+        }
+
+        list_for_each_entry_safe (snap_vol, tmp, &snap->volumes, vol_list) {
+                ret = glusterd_snap_volume_remove (rsp_dict, snap_vol,
+                                                   remove_lvm, force);
+                if (ret) {
+                        gf_log(this->name, GF_LOG_WARNING, "Failed to remove "
+                               "volinfo %s for snap %s", snap_vol->volname,
+                               snap->snapname);
+                        save_ret = ret;
+
+                        /* Continue to cleaning up the snap in case of error
+                           if force flag is enabled */
+                        if (!force)
+                                goto out;
+                }
+        }
+
+        ret = glusterd_store_delete_snap (snap);
+        if (ret) {
+                gf_log(this->name, GF_LOG_WARNING, "Failed to remove snap %s "
+                       "from store", snap->snapname);
+                save_ret = ret;
+                if (!force)
+                        goto out;
+        }
+
+        ret = glusterd_snapobject_delete (snap);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING, "Failed to delete "
+                        "snap object %s", snap->snapname);
+
+        if (save_ret)
+                ret = save_ret;
+out:
+        gf_log (THIS->name, GF_LOG_TRACE, "returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_snapshot_get_snapvol_detail (dict_t *dict,
+                                      glusterd_volinfo_t *snap_vol,
+                                      char *keyprefix, int detail)
+{
+        int                 ret           = -1;
+        int                 snap_limit    = 0;
+        char                key[PATH_MAX] = {0,};
+        char               *value         = NULL;
+        glusterd_volinfo_t *origin_vol    = NULL;
+        glusterd_conf_t    *conf          = NULL;
+        xlator_t           *this          = NULL;
+
+        this = THIS;
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GF_ASSERT (dict);
+        GF_ASSERT (snap_vol);
+        GF_ASSERT (keyprefix);
+
+        /* Volume Name */
+        value = gf_strdup (snap_vol->volname);
+        if (!value)
+                goto out;
+
+        snprintf (key, sizeof (key), "%s.volname", keyprefix);
+        ret = dict_set_dynstr (dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                        "volume name in dictionary: %s", key);
+                goto out;
+        }
+
+        /* Volume ID */
+        value = gf_strdup (uuid_utoa (snap_vol->volume_id));
+        if (NULL == value) {
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "%s.vol-id", keyprefix);
+        ret = dict_set_dynstr (dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                        "volume id in dictionary: %s", key);
+                goto out;
+        }
+        value = NULL;
+
+        /* volume status */
+        snprintf (key, sizeof (key), "%s.vol-status", keyprefix);
+        switch (snap_vol->status) {
+        case GLUSTERD_STATUS_STARTED:
+                ret = dict_set_str (dict, key, "Started");
+                break;
+        case GLUSTERD_STATUS_STOPPED:
+                ret = dict_set_str (dict, key, "Stopped");
+                break;
+        case GD_SNAP_STATUS_NONE:
+                ret = dict_set_str (dict, key, "None");
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR, "Invalid volume status");
+                ret = -1;
+                goto out;
+        }
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set volume status"
+                        " in dictionary: %s", key);
+                goto out;
+        }
+
+
+        ret = glusterd_volinfo_find (snap_vol->parent_volname, &origin_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to get the parent "
+                        "volinfo for the volume %s", snap_vol->volname);
+                goto out;
+        }
+
+        /* Snaps available */
+        if (conf->snap_max_hard_limit < origin_vol->snap_max_hard_limit) {
+                snap_limit = conf->snap_max_hard_limit;
+                gf_log(this->name, GF_LOG_DEBUG, "system snap-max-hard-limit is"
+                       " lesser than volume snap-max-hard-limit, "
+                       "snap-max-hard-limit value is set to %d", snap_limit);
+        } else {
+                snap_limit = origin_vol->snap_max_hard_limit;
+                gf_log(this->name, GF_LOG_DEBUG, "volume snap-max-hard-limit is"
+                       " lesser than system snap-max-hard-limit, "
+                       "snap-max-hard-limit value is set to %d", snap_limit);
+        }
+
+        snprintf (key, sizeof (key), "%s.snaps-available", keyprefix);
+        if (snap_limit > origin_vol->snap_count)
+                ret = dict_set_int32 (dict, key,
+                        snap_limit - origin_vol->snap_count);
+        else
+                ret = dict_set_int32 (dict, key, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set available snaps");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "%s.snapcount", keyprefix);
+        ret = dict_set_int32 (dict, key, origin_vol->snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not save snapcount");
+                goto out;
+        }
+
+        if (!detail)
+                goto out;
+
+        /* Parent volume name */
+        value = gf_strdup (snap_vol->parent_volname);
+        if (!value)
+                goto out;
+
+        snprintf (key, sizeof (key), "%s.origin-volname", keyprefix);
+        ret = dict_set_dynstr (dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set parent "
+                        "volume name in dictionary: %s", key);
+                goto out;
+        }
+        value = NULL;
+
+        ret = 0;
+out:
+        if (value)
+                GF_FREE (value);
+
+        return ret;
+}
+
+static int
+glusterd_snapshot_get_snap_detail (dict_t *dict, glusterd_snap_t *snap,
+                                char *keyprefix, glusterd_volinfo_t *volinfo)
+{
+        int                 ret           = -1;
+        int                 volcount      = 0;
+        char                key[PATH_MAX] = {0,};
+        char               *value         = NULL;
+        char               *timestr       = NULL;
+        struct tm          *tmptr         = NULL;
+        glusterd_volinfo_t *snap_vol      = NULL;
+        glusterd_volinfo_t *tmp_vol       = NULL;
+        xlator_t           *this          = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (snap);
+        GF_ASSERT (keyprefix);
+
+        /* Snap Name */
+        value = gf_strdup (snap->snapname);
+        if (!value)
+                goto out;
+
+        snprintf (key, sizeof (key), "%s.snapname", keyprefix);
+        ret = dict_set_dynstr (dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                        "snap name in dictionary");
+                goto out;
+        }
+
+        /* Snap ID */
+        value = gf_strdup (uuid_utoa (snap->snap_id));
+        if (NULL == value) {
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "%s.snap-id", keyprefix);
+        ret = dict_set_dynstr (dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                "snap id in dictionary");
+                goto out;
+        }
+        value = NULL;
+
+        tmptr = localtime (&(snap->time_stamp));
+        if (NULL == tmptr) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to convert "
+                                "time_t to *tm");
+                ret = -1;
+                goto out;
+        }
+
+        timestr = GF_CALLOC (1, PATH_MAX, gf_gld_mt_char);
+        if (NULL == timestr) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = strftime (timestr, PATH_MAX, "%Y-%m-%d %H:%M:%S", tmptr);
+        if (0 == ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to convert time_t "
+                        "to string");
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "%s.snap-time", keyprefix);
+        ret = dict_set_dynstr (dict, key, timestr);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                "snap time stamp in dictionary");
+                goto out;
+        }
+        timestr = NULL;
+
+        /* If snap description is provided then add that into dictionary */
+        if (NULL != snap->description) {
+                value = gf_strdup (snap->description);
+                if (NULL == value) {
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (key, sizeof (key), "%s.snap-desc", keyprefix);
+                ret = dict_set_dynstr (dict, key, value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                        "snap description in dictionary");
+                        goto out;
+                }
+                value = NULL;
+        }
+
+        snprintf (key, sizeof (key), "%s.snap-status", keyprefix);
+        switch (snap->snap_status) {
+        case GD_SNAP_STATUS_INIT:
+                ret = dict_set_str (dict, key, "Init");
+                break;
+        case GD_SNAP_STATUS_IN_USE:
+                ret = dict_set_str (dict, key, "In-use");
+                break;
+        case GD_SNAP_STATUS_DECOMMISSION:
+                ret = dict_set_str (dict, key, "Decommisioned");
+                break;
+        case GD_SNAP_STATUS_RESTORED:
+                ret = dict_set_str (dict, key, "Restored");
+                break;
+        case GD_SNAP_STATUS_NONE:
+                ret = dict_set_str (dict, key, "None");
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR, "Invalid snap status");
+                ret = -1;
+                goto out;
+        }
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snap status "
+                        "in dictionary");
+                goto out;
+        }
+
+        if (volinfo) {
+                volcount = 1;
+                snprintf (key, sizeof (key), "%s.vol%d", keyprefix, volcount);
+                ret = glusterd_snapshot_get_snapvol_detail (dict,
+                                                        volinfo, key, 0);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "get volume detail %s for snap %s",
+                                snap_vol->volname, snap->snapname);
+                        goto out;
+                }
+                goto done;
+        }
+
+        list_for_each_entry_safe (snap_vol, tmp_vol, &snap->volumes, vol_list) {
+                volcount++;
+                snprintf (key, sizeof (key), "%s.vol%d", keyprefix, volcount);
+                ret = glusterd_snapshot_get_snapvol_detail (dict,
+                                                        snap_vol, key, 1);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "get volume detail %s for snap %s",
+                                snap_vol->volname, snap->snapname);
+                        goto out;
+                }
+        }
+
+done:
+        snprintf (key, sizeof (key), "%s.vol-count", keyprefix);
+        ret = dict_set_int32 (dict, key, volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set %s",
+                        key);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (value)
+                GF_FREE (value);
+
+        if (timestr)
+                GF_FREE(timestr);
+
+        return ret;
+}
+
+static int
+glusterd_snapshot_get_all_snap_info (dict_t *dict)
+{
+        int                 ret           = -1;
+        int                 snapcount     = 0;
+        char                key[PATH_MAX] = {0,};
+        glusterd_snap_t    *snap          = NULL;
+        glusterd_snap_t    *tmp_snap      = NULL;
+        glusterd_conf_t    *priv          = NULL;
+        xlator_t           *this          = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* General parameter validation */
+        GF_ASSERT (dict);
+
+        list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots, snap_list) {
+                snapcount++;
+                snprintf (key, sizeof (key), "snap%d", snapcount);
+                ret = glusterd_snapshot_get_snap_detail (dict, snap, key, NULL);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "snapdetail for snap %s", snap->snapname);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (dict, "snap-count", snapcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_get_info_by_volume (dict_t *dict, char *volname,
+                                      char *err_str, size_t len)
+{
+        int                  ret           = -1;
+        int                  snapcount     = 0;
+        int                  snap_limit    = 0;
+        char                *value         = NULL;
+        char                 key[PATH_MAX] = "";
+        glusterd_volinfo_t  *volinfo       = NULL;
+        glusterd_volinfo_t  *snap_vol      = NULL;
+        glusterd_volinfo_t  *tmp_vol       = NULL;
+        glusterd_conf_t     *conf          = NULL;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GF_ASSERT (dict);
+        GF_ASSERT (volname);
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (err_str, len, "Volume (%s) does not exist", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        /* Snaps available */
+        if (conf->snap_max_hard_limit < volinfo->snap_max_hard_limit) {
+                snap_limit = conf->snap_max_hard_limit;
+                gf_log(this->name, GF_LOG_DEBUG, "system snap-max-hard-limit is"
+                       " lesser than volume snap-max-hard-limit, "
+                       "snap-max-hard-limit value is set to %d", snap_limit);
+        } else {
+                snap_limit = volinfo->snap_max_hard_limit;
+                gf_log(this->name, GF_LOG_DEBUG, "volume snap-max-hard-limit is"
+                       " lesser than system snap-max-hard-limit, "
+                       "snap-max-hard-limit value is set to %d", snap_limit);
+        }
+
+        if (snap_limit > volinfo->snap_count)
+                ret = dict_set_int32 (dict, "snaps-available",
+                        snap_limit - volinfo->snap_count);
+        else
+                ret = dict_set_int32 (dict, "snaps-available", 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set available snaps");
+                goto out;
+        }
+
+        /* Origin volume name */
+        value = gf_strdup (volinfo->volname);
+        if (!value)
+                goto out;
+
+        ret = dict_set_dynstr (dict, "origin-volname", value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set parent "
+                        "volume name in dictionary: %s", key);
+                goto out;
+        }
+        value = NULL;
+
+        list_for_each_entry_safe (snap_vol, tmp_vol, &volinfo->snap_volumes,
+                                  snapvol_list) {
+                snapcount++;
+                snprintf (key, sizeof (key), "snap%d", snapcount);
+                ret = glusterd_snapshot_get_snap_detail (dict,
+                                                         snap_vol->snapshot,
+                                                         key, snap_vol);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "snapdetail for snap %s",
+                                snap_vol->snapshot->snapname);
+                        goto out;
+                }
+        }
+        ret = dict_set_int32 (dict, "snap-count", snapcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (value)
+                GF_FREE (value);
+
+        return ret;
+}
+
+/* This function will be called from RPC handler routine.
+ * This function is responsible for getting the requested
+ * snapshot info into the dictionary.
+ *
+ * @param req   RPC request object. Required for sending a response back.
+ * @param op    glusterd operation. Required for sending a response back.
+ * @param dict  pointer to dictionary which will contain both
+ *              request and response key-pair values.
+ * @return -1 on error and 0 on success
+ */
+int
+glusterd_handle_snapshot_info (rpcsvc_request_t *req, glusterd_op_t op,
+                               dict_t *dict, char *err_str, size_t len)
+{
+        int              ret             =       -1;
+        int8_t          snap_driven      =        1;
+        char            *volname         =      NULL;
+        char            *snapname        =      NULL;
+        glusterd_snap_t *snap            =      NULL;
+        xlator_t        *this            =      NULL;
+        int32_t         cmd              =      GF_SNAP_INFO_TYPE_ALL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_VALIDATE_OR_GOTO (this->name, req, out);
+        GF_VALIDATE_OR_GOTO (this->name, dict, out);
+
+
+        ret = dict_get_int32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get type "
+                        "of snapshot info");
+                goto out;
+        }
+
+        switch (cmd) {
+                case GF_SNAP_INFO_TYPE_ALL:
+                {
+                        ret = glusterd_snapshot_get_all_snap_info (dict);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get info of all snaps");
+                                goto out;
+                        }
+                        break;
+                }
+
+                case GF_SNAP_INFO_TYPE_SNAP:
+                {
+                        ret = dict_get_str (dict, "snapname", &snapname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get snap name");
+                                goto out;
+                        }
+
+                        ret = dict_set_int32 (dict, "snap-count", 1);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set snapcount");
+                                goto out;
+                        }
+
+                        snap = glusterd_find_snap_by_name (snapname);
+                        if (!snap) {
+                                snprintf (err_str, len,
+                                         "Snap (%s) does not exist", snapname);
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "%s", err_str);
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = glusterd_snapshot_get_snap_detail (dict, snap,
+                                                                "snap1", NULL);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get snap detail of snap "
+                                        "%s", snap->snapname);
+                                goto out;
+                        }
+                        break;
+                }
+
+                case GF_SNAP_INFO_TYPE_VOL:
+                {
+                        ret = dict_get_str (dict, "volname", &volname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get volname");
+                                goto out;
+                        }
+                        ret = glusterd_snapshot_get_info_by_volume (dict,
+                                                        volname, err_str, len);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get volume info of volume "
+                                        "%s", volname);
+                                goto out;
+                        }
+                        snap_driven = 0;
+                        break;
+                }
+        }
+
+        ret = dict_set_int8 (dict, "snap-driven", snap_driven);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snap-driven");
+                goto out;
+        }
+
+        /* If everything is successful then send the response back to cli.
+         * In case of failure the caller of this function will take care
+           of the response */
+        ret = glusterd_op_send_cli_response (op, 0, 0, req, dict, err_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to send cli "
+                                "response");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+/* This function sets all the snapshot names in the dictionary */
+int
+glusterd_snapshot_get_all_snapnames (dict_t *dict)
+{
+        int              ret             = -1;
+        int              snapcount       = 0;
+        char            *snapname        = NULL;
+        char             key[PATH_MAX]   = {0,};
+        glusterd_snap_t *snap            = NULL;
+        glusterd_snap_t *tmp_snap        = NULL;
+        glusterd_conf_t *priv            = NULL;
+        xlator_t        *this            = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (dict);
+
+        list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots, snap_list) {
+                snapcount++;
+                snapname = gf_strdup (snap->snapname);
+                if (!snapname) {
+                        gf_log (this->name, GF_LOG_ERROR, "strdup failed");
+                        ret = -1;
+                        goto out;
+                }
+                snprintf (key, sizeof (key), "snapname%d", snapcount);
+                ret = dict_set_dynstr (dict, key, snapname);
+                if (ret) {
+                        GF_FREE (snapname);
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set %s",
+                                key);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (dict, "snap-count", snapcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+
+        return ret;
+}
+
+/* This function sets all the snapshot names
+   under a given volume in the dictionary */
+int
+glusterd_snapshot_get_vol_snapnames (dict_t *dict, glusterd_volinfo_t *volinfo)
+{
+        int                 ret             = -1;
+        int                 snapcount       = 0;
+        char               *snapname        = NULL;
+        char                key[PATH_MAX]   = {0,};
+        glusterd_volinfo_t *snap_vol        = NULL;
+        glusterd_volinfo_t *tmp_vol         = NULL;
+        xlator_t           *this            = NULL;
+
+        this = THIS;
+        GF_ASSERT (dict);
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry_safe (snap_vol, tmp_vol,
+                                  &volinfo->snap_volumes, snapvol_list) {
+                snapcount++;
+                snapname = gf_strdup (snap_vol->snapshot->snapname);
+                if (!snapname) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "strdup failed");
+                        ret = -1;
+                        goto out;
+                }
+                snprintf (key, sizeof (key), "snapname%d", snapcount);
+                ret = dict_set_dynstr (dict, key, snapname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "set %s", key);
+                        GF_FREE (snapname);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (dict, "snap-count", snapcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+
+        return ret;
+}
+
+int
+glusterd_handle_snapshot_list (rpcsvc_request_t *req, glusterd_op_t op,
+                               dict_t *dict, char *err_str, size_t len)
+{
+        int                 ret             = -1;
+        char               *volname         = NULL;
+        glusterd_volinfo_t *volinfo         = NULL;
+        xlator_t           *this            = NULL;
+
+        this = THIS;
+
+        GF_VALIDATE_OR_GOTO (this->name, req, out);
+        GF_VALIDATE_OR_GOTO (this->name, dict, out);
+
+        /* Ignore error for getting volname as it is optional */
+        ret = dict_get_str (dict, "volname", &volname);
+
+        if (NULL == volname) {
+                ret = glusterd_snapshot_get_all_snapnames (dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to get snapshot list");
+                        goto out;
+                }
+        } else {
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (err_str, len,
+                                 "Volume (%s) does not exist", volname);
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s", err_str);
+                        goto out;
+                }
+
+                ret = glusterd_snapshot_get_vol_snapnames (dict, volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to get snapshot list for volume %s",
+                                volname);
+                        goto out;
+                }
+        }
+
+        /* If everything is successful then send the response back to cli.
+        In case of failure the caller of this function will take of response.*/
+        ret = glusterd_op_send_cli_response (op, 0, 0, req, dict, err_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to send cli "
+                                "response");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+/* This is a snapshot create handler function. This function will be
+ * executed in the originator node. This function is responsible for
+ * calling mgmt_v3 framework to do the actual snap creation on all the bricks
+ *
+ * @param req           RPC request object
+ * @param op            gluster operation
+ * @param dict          dictionary containing snapshot restore request
+ * @param err_str       In case of an err this string should be populated
+ * @param len           length of err_str buffer
+ *
+ * @return              Negative value on Failure and 0 in success
+ */
+int
+glusterd_handle_snapshot_create (rpcsvc_request_t *req, glusterd_op_t op,
+                               dict_t *dict, char *err_str, size_t len)
+{
+        int           ret                     = -1;
+        char         *volname                 = NULL;
+        char         *snapname                = NULL;
+        int64_t       volcount                = 0;
+        xlator_t     *this                    = NULL;
+        char          key[PATH_MAX]           = "";
+        char         *username                = NULL;
+        char         *password                = NULL;
+        uuid_t       *uuid_ptr                = NULL;
+        uuid_t        tmp_uuid                = {0};
+        int           i                       = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        GF_ASSERT (err_str);
+
+        ret = dict_get_int64 (dict, "volcount", &volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                        "get the volume count");
+                goto out;
+        }
+        if (volcount <= 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Invalid volume count %ld "
+                        "supplied", volcount);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to get the snapname");
+                goto out;
+        }
+
+        if (strlen(snapname) >= GLUSTERD_MAX_SNAP_NAME) {
+                snprintf (err_str, len, "snapname cannot exceed 255 "
+                          "characters");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                ret = -1;
+                goto out;
+        }
+
+        uuid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!uuid_ptr) {
+                gf_log (this->name, GF_LOG_ERROR, "Out Of Memory");
+                ret = -1;
+                goto out;
+        }
+
+        uuid_generate (*uuid_ptr);
+        ret = dict_set_bin (dict, "snap-id", uuid_ptr, sizeof(uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to set snap-id");
+                GF_FREE (uuid_ptr);
+                goto out;
+        }
+        uuid_ptr = NULL;
+
+        ret = dict_set_int64 (dict, "snap-time", (int64_t)time(NULL));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to set snap-time");
+                goto out;
+        }
+
+        for (i = 1; i <= volcount; i++) {
+                snprintf (key, sizeof (key), "volname%d", i);
+                ret = dict_get_str (dict, key, &volname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to get volume name");
+                        goto out;
+                }
+
+                /* generate internal username and password  for the snap*/
+                uuid_generate (tmp_uuid);
+                username = gf_strdup (uuid_utoa (tmp_uuid));
+                snprintf (key, sizeof(key), "volume%d_username", i);
+                ret = dict_set_dynstr (dict, key, username);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
+                                "username for volume %s", volname);
+                        GF_FREE (username);
+                        goto out;
+                }
+
+                uuid_generate (tmp_uuid);
+                password = gf_strdup (uuid_utoa (tmp_uuid));
+                snprintf (key, sizeof(key), "volume%d_password", i);
+                ret = dict_set_dynstr (dict, key, password);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
+                                "password for volume %s", volname);
+                        GF_FREE (password);
+                        goto out;
+                }
+
+                uuid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+                if (!uuid_ptr) {
+                        gf_log (this->name, GF_LOG_ERROR, "Out Of Memory");
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (key, sizeof(key) - 1, "vol%d_volid", i);
+                uuid_generate (*uuid_ptr);
+                ret = dict_set_bin (dict, key, uuid_ptr, sizeof(uuid_t));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to set snap_volid");
+                        GF_FREE (uuid_ptr);
+                        goto out;
+                }
+        }
+
+        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate snap "
+                                "phases");
+        }
+
+out:
+        return ret;
+}
+
+/* This is a snapshot status handler function. This function will be
+ * executed in a originator node. This function is responsible for
+ * calling mgmt v3 framework to get the actual snapshot status from
+ * all the bricks
+ *
+ * @param req           RPC request object
+ * @param op            gluster operation
+ * @param dict          dictionary containing snapshot status request
+ * @param err_str       In case of an err this string should be populated
+ * @param len           length of err_str buffer
+ *
+ * return :  0  in case of success.
+ *          -1  in case of failure.
+ *
+ */
+int
+glusterd_handle_snapshot_status (rpcsvc_request_t *req, glusterd_op_t op,
+                                 dict_t *dict, char *err_str, size_t len)
+{
+        int                     ret             =       -1;
+        char                    *volname        =       NULL;
+        char                    *snapname       =       NULL;
+        char                    *buf            =       NULL;
+        glusterd_conf_t         *conf           =       NULL;
+        xlator_t                *this           =       NULL;
+        int32_t                 cmd             =       -1;
+        int                     i               =       0;
+        dict_t                  *voldict        =       NULL;
+        char                    key[PATH_MAX]   =       "";
+        glusterd_volinfo_t     *volinfo         =       NULL;
+        glusterd_snap_t         *snap           =       NULL;
+        glusterd_volinfo_t      *snap_volinfo   =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+
+        GF_ASSERT (conf);
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        GF_ASSERT (err_str);
+
+        ret = dict_get_int32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not get status type");
+                goto out;
+        }
+        switch (cmd) {
+                case GF_SNAP_STATUS_TYPE_ALL:
+                {
+                        /* IF we give "gluster snapshot status"
+                         * then lock is held on all snaps.
+                         * This is the place where necessary information
+                         * (snapname and snapcount)is populated in dictionary
+                         * for locking.
+                         */
+                        ++i;
+                        list_for_each_entry (snap, &conf->snapshots, snap_list)
+                        {
+                                snprintf (key, sizeof (key), "snapname%d", i);
+                                buf = gf_strdup (snap->snapname);
+                                if (!buf) {
+                                        ret = -1;
+                                        goto out;
+                                }
+                                ret = dict_set_dynstr (dict, key, buf);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Could not save snapname (%s) "
+                                                "in the dictionary",
+                                                snap->snapname);
+                                        GF_FREE (buf);
+                                        goto out;
+                                }
+
+                                buf = NULL;
+                                i++;
+                        }
+
+                        ret = dict_set_int32 (dict, "snapcount", i - 1);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Could not "
+                                        "save snapcount in the dictionary");
+                                goto out;
+                        }
+                        break;
+                }
+
+                case GF_SNAP_STATUS_TYPE_SNAP:
+                {
+                        /* IF we give "gluster snapshot status <snapname>"
+                         * then lock is held on single snap.
+                         * This is the place where necessary information
+                         * (snapname)is populated in dictionary
+                         * for locking.
+                         */
+                        ret = dict_get_str (dict, "snapname", &snapname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to fetch snap name");
+                                goto out;
+                        }
+
+                        snap = glusterd_find_snap_by_name (snapname);
+                        if (!snap) {
+                                snprintf (err_str, len, "Snap (%s)"
+                                          "does not exist", snapname);
+                                gf_log(this->name, GF_LOG_ERROR,
+                                        "%s", err_str);
+                                ret = -1;
+                                goto out;
+                        }
+                        break;
+                }
+                case GF_SNAP_STATUS_TYPE_VOL:
+                        ret = dict_get_str (dict, "volname", &volname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to fetch volname");
+                                goto out;
+                        }
+
+                        ret = glusterd_volinfo_find (volname, &volinfo);
+                        if (ret) {
+                                snprintf (err_str, len, "Volume (%s) "
+                                          "does not exist", volname);
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "%s", err_str);
+                                goto out;
+                        }
+
+                        i = 1;
+                        list_for_each_entry (snap_volinfo,
+                                        &volinfo->snap_volumes, snapvol_list) {
+                                snprintf (key, sizeof (key), "snapname%d", i);
+
+                                buf = gf_strdup
+                                             (snap_volinfo->snapshot->snapname);
+                                if (!buf) {
+                                        ret = -1;
+                                        goto out;
+                                }
+
+                                ret = dict_set_dynstr (dict, key, buf);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Could not save snapname");
+                                        GF_FREE (buf);
+                                        goto out;
+                                }
+
+                                buf = NULL;
+                                i++;
+                        }
+
+                        ret = dict_set_int32 (dict, "snapcount", i-1);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not save snapcount");
+                                goto out;
+                        }
+                        break;
+                default:
+                {
+                        gf_log (this->name, GF_LOG_ERROR, "Unknown type");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        /* Volume lock is not necessary for snapshot status, hence
+         * turning it off
+         */
+        ret = dict_set_int8 (dict, "hold_vol_locks", 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Setting volume lock "
+                        "flag failed");
+                goto out;
+        }
+
+        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate "
+                        "snap phases");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        if (voldict) {
+                dict_unref (voldict);
+        }
+        return ret;
+}
+
+
+/* This is a snapshot restore handler function. This function will be
+ * executed in the originator node. This function is responsible for
+ * calling mgmt_v3 framework to do the actual restore on all the bricks
+ *
+ * @param req           RPC request object
+ * @param op            gluster operation
+ * @param dict          dictionary containing snapshot restore request
+ * @param err_str       In case of an err this string should be populated
+ * @param len           length of err_str buffer
+ *
+ * @return              Negative value on Failure and 0 in success
+ */
+int
+glusterd_handle_snapshot_restore (rpcsvc_request_t *req, glusterd_op_t op,
+                               dict_t *dict, char *err_str, size_t len)
+{
+        int                     ret             = -1;
+        char                    *snapname       = NULL;
+        char                    *buf            = NULL;
+        glusterd_conf_t         *conf           = NULL;
+        xlator_t                *this           = NULL;
+        glusterd_snap_t         *snap           = NULL;
+        glusterd_volinfo_t      *snap_volinfo   = NULL;
+        int32_t                 i               =    0;
+        char                    key[PATH_MAX]   =    "";
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+
+        GF_ASSERT (conf);
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        GF_ASSERT (err_str);
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                        "get snapname");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (!snap) {
+                snprintf (err_str, len, "Snap (%s) does not exist", snapname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                ret = -1;
+                goto out;
+        }
+
+        list_for_each_entry (snap_volinfo, &snap->volumes, vol_list) {
+                i++;
+                snprintf (key, sizeof (key), "volname%d", i);
+                buf = gf_strdup (snap_volinfo->parent_volname);
+                if (!buf) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_dynstr (dict, key, buf);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Could not set "
+                                "parent volume name %s in the dict",
+                                snap_volinfo->parent_volname);
+                        GF_FREE (buf);
+                        goto out;
+                }
+                buf = NULL;
+        }
+
+        ret = dict_set_int32 (dict, "volcount", i);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not save volume count");
+                goto out;
+        }
+
+        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate snap "
+                                "phases");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+glusterd_snap_t*
+glusterd_create_snap_object (dict_t *dict, dict_t *rsp_dict)
+{
+        char              *snapname    = NULL;
+        uuid_t            *snap_id     = NULL;
+        char              *description = NULL;
+        glusterd_snap_t   *snap        = NULL;
+        xlator_t          *this        = NULL;
+        glusterd_conf_t   *priv        = NULL;
+        int                ret         = -1;
+        int64_t            time_stamp  = 0;
+
+        this = THIS;
+        priv = this->private;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        /* Fetch snapname, description, id and time from dict */
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snapname");
+                goto out;
+        }
+
+        /* Ignore ret value for description*/
+        ret = dict_get_str (dict, "description", &description);
+
+        ret = dict_get_bin (dict, "snap-id", (void **)&snap_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snap_id");
+                goto out;
+        }
+
+        ret = dict_get_int64 (dict, "snap-time", &time_stamp);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snap-time");
+                goto out;
+        }
+        if (time_stamp <= 0) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Invalid time-stamp: %ld",
+                        time_stamp);
+                goto out;
+        }
+
+        list_for_each_entry (snap, &priv->snapshots, snap_list) {
+                if (!strcmp (snap->snapname, snapname) ||
+                    !uuid_compare (snap->snap_id, *snap_id)) {
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "Found duplicate snap %s (%s)",
+                                snap->snapname, uuid_utoa (snap->snap_id));
+                        ret = -1;
+                        break;
+                }
+        }
+        if (ret) {
+                snap = NULL;
+                goto out;
+        }
+
+        snap = glusterd_new_snap_object ();
+        if (!snap) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not create "
+                        "the snap object for snap %s", snapname);
+                goto out;
+        }
+
+        strcpy (snap->snapname, snapname);
+        uuid_copy (snap->snap_id, *snap_id);
+        snap->time_stamp = (time_t)time_stamp;
+        /* Set the status as GD_SNAP_STATUS_INIT and once the backend snapshot
+           is taken and snap is really ready to use, set the status to
+           GD_SNAP_STATUS_IN_USE. This helps in identifying the incomplete
+           snapshots and cleaning them up.
+        */
+        snap->snap_status = GD_SNAP_STATUS_INIT;
+        if (description) {
+                snap->description = gf_strdup (description);
+                if (snap->description == NULL) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Saving the Snap Description Failed");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = glusterd_store_snap (snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Could not store snap"
+                        "object %s", snap->snapname);
+                goto out;
+        }
+
+        list_add_order (&snap->snap_list, &priv->snapshots,
+                        glusterd_compare_snap_time);
+
+        gf_log (this->name, GF_LOG_TRACE, "Snap %s added to the list",
+                snap->snapname);
+
+        ret = 0;
+
+out:
+        if (ret) {
+                if (snap)
+                        glusterd_snap_remove (rsp_dict, snap,
+                                              _gf_true, _gf_true);
+                snap = NULL;
+        }
+
+        return snap;
+}
+
+/* This function is called to get the device path of the snap lvm. Usually
+   if /dev/mapper/<group-name>-<lvm-name> is the device for the lvm,
+   then the snap device will be /dev/<group-name>/<snapname>.
+   This function takes care of building the path for the snap device.
+*/
+char *
+glusterd_build_snap_device_path (char *device, char *snapname)
+{
+        char        snap[PATH_MAX]      = "";
+        char        msg[1024]           = "";
+        char        volgroup[PATH_MAX]  = "";
+        char       *snap_device         = NULL;
+        xlator_t   *this                = NULL;
+        runner_t    runner              = {0,};
+        char       *ptr                 = NULL;
+        int         ret                 = -1;
+
+        this = THIS;
+        GF_ASSERT (this);
+        if (!device) {
+                gf_log (this->name, GF_LOG_ERROR, "device is NULL");
+                goto out;
+        }
+        if (!snapname) {
+                gf_log (this->name, GF_LOG_ERROR, "snapname is NULL");
+                goto out;
+        }
+
+        runinit (&runner);
+        runner_add_args (&runner, "/sbin/lvs", "--noheadings", "-o", "vg_name",
+                         device, NULL);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        snprintf (msg, sizeof (msg), "Get volume group for device %s", device);
+        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
+        ret = runner_start (&runner);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume group "
+                        "for device %s", device);
+                runner_end (&runner);
+                goto out;
+        }
+        ptr = fgets(volgroup, sizeof(volgroup),
+                    runner_chio (&runner, STDOUT_FILENO));
+        if (!ptr || !strlen(volgroup)) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume group "
+                        "for snap %s", snapname);
+                runner_end (&runner);
+                ret = -1;
+                goto out;
+        }
+        runner_end (&runner);
+
+        snprintf (snap, sizeof(snap), "/dev/%s/%s", gf_trim(volgroup),
+                  snapname);
+        snap_device = gf_strdup (snap);
+        if (!snap_device) {
+                gf_log (this->name, GF_LOG_WARNING, "Cannot copy the "
+                        "snapshot device name for snapname: %s)", snapname);
+        }
+
+out:
+        return snap_device;
+}
+
+/* This function actually calls the command (or the API) for taking the
+   snapshot of the backend brick filesystem. If this is successful,
+   then call the glusterd_snap_create function to create the snap object
+   for glusterd
+*/
+char *
+glusterd_take_lvm_snapshot (glusterd_volinfo_t *snap_vol,
+                            glusterd_brickinfo_t *brickinfo)
+{
+        char             msg[NAME_MAX]    = "";
+        char             buf[PATH_MAX]    = "";
+        char            *snap_device      = NULL;
+        char            *ptr              = NULL;
+        char            *device           = NULL;
+        int              ret              = -1;
+        gf_boolean_t     match            = _gf_false;
+        runner_t         runner           = {0,};
+        xlator_t        *this             = NULL;
+
+        this = THIS;
+
+        if (!brickinfo) {
+                gf_log (this->name, GF_LOG_ERROR, "brickinfo NULL");
+                goto out;
+        }
+
+        device = glusterd_get_brick_mount_details (brickinfo);
+        if (!device) {
+                gf_log (this->name, GF_LOG_ERROR, "getting device name for "
+                        "the brick %s:%s failed", brickinfo->hostname,
+                        brickinfo->path);
+                goto out;
+        }
+
+        /* Figuring out if setactivationskip flag is supported or not */
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "running lvcreate help");
+        runner_add_args (&runner, "/sbin/lvcreate", "--help", NULL);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        ret = runner_start (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to run lvcreate help");
+                runner_end (&runner);
+                goto out;
+        }
+
+        /* Looking for setactivationskip in lvcreate --help */
+        do {
+                ptr = fgets(buf, sizeof(buf),
+                            runner_chio (&runner, STDOUT_FILENO));
+                if (ptr) {
+                        if (strstr(buf, "setactivationskip")) {
+                                match = _gf_true;
+                                break;
+                        }
+                }
+        } while (ptr != NULL);
+        runner_end (&runner);
+
+        /* Takng the actual snapshot */
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "taking snapshot of the brick %s:%s",
+                  brickinfo->hostname, brickinfo->path);
+        if (match == _gf_true)
+                runner_add_args (&runner, "/sbin/lvcreate", "-s", device,
+                                 "--setactivationskip", "n", "--name",
+                                 snap_vol->volname, NULL);
+        else
+                runner_add_args (&runner, "/sbin/lvcreate", "-s", device,
+                                 "--name", snap_vol->volname, NULL);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+        ret = runner_start (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "taking snapshot of the "
+                        "brick (%s:%s) of device %s failed",
+                        brickinfo->hostname, brickinfo->path, device);
+                runner_end (&runner);
+                goto out;
+        }
+        runner_end (&runner);
+
+        snap_device = glusterd_build_snap_device_path (device,
+                                                       snap_vol->volname);
+        if (!snap_device) {
+                gf_log (this->name, GF_LOG_WARNING, "Cannot copy the snapshot "
+                        "device name for snap %s (volume id: %s)",
+                        snap_vol->snapshot->snapname, snap_vol->volname);
+                ret = -1;
+                goto out;
+        }
+
+out:
+        return snap_device;
+}
+
+int32_t
+glusterd_snap_brick_create (char *device, glusterd_volinfo_t *snap_volinfo,
+                            glusterd_brickinfo_t *original_brickinfo,
+                            int32_t brick_count, char *snap_brick_dir)
+{
+        int32_t          ret                             = -1;
+        xlator_t        *this                            = NULL;
+        glusterd_conf_t *priv                            = NULL;
+        char             snap_brick_mount_path[PATH_MAX] = "";
+        char             snap_brick_path[PATH_MAX]       = "";
+        char             msg[1024]                       = "";
+        struct stat      statbuf                         = {0, };
+        runner_t         runner                          = {0, };
+
+        this = THIS;
+        priv = this->private;
+
+        GF_ASSERT (device);
+        GF_ASSERT (snap_volinfo);
+        GF_ASSERT (original_brickinfo);
+        GF_ASSERT (snap_brick_dir);
+
+        snprintf (snap_brick_mount_path, sizeof (snap_brick_mount_path),
+                  "%s/%s/brick%d",  snap_mount_folder, snap_volinfo->volname,
+                  brick_count+1);
+
+        snprintf (snap_brick_path, sizeof (snap_brick_path), "%s%s",
+                  snap_brick_mount_path, snap_brick_dir);
+
+        ret = mkdir_p (snap_brick_mount_path, 0777, _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "creating the brick directory"
+                        " %s for the snapshot %s(device: %s) failed",
+                        snap_brick_mount_path, snap_volinfo->volname, device);
+                goto out;
+        }
+        /* mount the snap logical device on the directory inside
+           /run/gluster/snaps/<snapname>/@snap_brick_mount_path
+           Way to mount the snap brick via mount api is this.
+           ret = mount (device, snap_brick_mount_path, entry->mnt_type,
+                        MS_MGC_VAL, "nouuid");
+           But for now, mounting using runner apis.
+        */
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "mounting snapshot of the brick %s:%s",
+                  original_brickinfo->hostname, original_brickinfo->path);
+        runner_add_args (&runner, "mount", "-o", "nouuid", device,
+                         snap_brick_mount_path, NULL);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+
+        /* let glusterd get blocked till snapshot is over */
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "mounting the snapshot "
+                        "logical device %s failed (error: %s)", device,
+                        strerror (errno));
+                goto out;
+        } else
+                gf_log (this->name, GF_LOG_DEBUG, "mounting the snapshot "
+                        "logical device %s successful", device);
+
+        ret = stat (snap_brick_path, &statbuf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "stat of the brick %s"
+                        "(brick mount: %s) failed (%s)", snap_brick_path,
+                        snap_brick_mount_path, strerror (errno));
+                goto out;
+        }
+        ret = sys_lsetxattr (snap_brick_path,
+                             GF_XATTR_VOL_ID_KEY,
+                             snap_volinfo->volume_id, 16,
+                             XATTR_REPLACE);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                        "extended attribute %s on %s. Reason: "
+                        "%s, snap: %s", GF_XATTR_VOL_ID_KEY,
+                        snap_brick_path, strerror (errno),
+                        snap_volinfo->volname);
+                goto out;
+        }
+
+out:
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "unmounting the snap brick"
+                        " mount %s", snap_brick_mount_path);
+                umount (snap_brick_mount_path);
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Added missed_snap_entry to rsp_dict */
+int32_t
+glusterd_add_missed_snaps_to_dict (dict_t *rsp_dict, char *snap_uuid,
+                                   glusterd_brickinfo_t *brickinfo,
+                                   int32_t brick_number, int32_t op)
+{
+        char                   *buf                             = NULL;
+        char                    missed_snap_entry[PATH_MAX]     = "";
+        char                    name_buf[PATH_MAX]              = "";
+        int32_t                 missed_snap_count               = -1;
+        int32_t                 ret                             = -1;
+        xlator_t               *this                            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap_uuid);
+        GF_ASSERT (brickinfo);
+
+        snprintf (missed_snap_entry, sizeof(missed_snap_entry),
+                  "%s:%s=%d:%s:%d:%d", uuid_utoa(brickinfo->uuid),
+                  snap_uuid, brick_number, brickinfo->path, op,
+                  GD_MISSED_SNAP_PENDING);
+
+        buf = gf_strdup (missed_snap_entry);
+        if (!buf) {
+                ret = -1;
+                goto out;
+        }
+
+        /* Fetch the missed_snap_count from the dict */
+        ret = dict_get_int32 (rsp_dict, "missed_snap_count",
+                              &missed_snap_count);
+        if (ret) {
+                /* Initialize the missed_snap_count for the first time */
+                missed_snap_count = 0;
+        }
+
+        /* Setting the missed_snap_entry in the rsp_dict */
+        snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
+                  missed_snap_count);
+        ret = dict_set_dynstr (rsp_dict, name_buf, buf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set missed_snap_entry (%s) "
+                        "in the rsp_dict.", buf);
+                GF_FREE (buf);
+                goto out;
+        }
+        missed_snap_count++;
+
+        /* Setting the new missed_snap_count in the dict */
+        ret = dict_set_int32 (rsp_dict, "missed_snap_count",
+                              missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set missed_snap_count for %s "
+                        "in the rsp_dict.", missed_snap_entry);
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int32_t
+glusterd_add_bricks_to_snap_volume (dict_t *dict, dict_t *rsp_dict,
+                                    glusterd_volinfo_t  *snap_vol,
+                                    glusterd_brickinfo_t *original_brickinfo,
+                                    glusterd_brickinfo_t *snap_brickinfo,
+                                    char **snap_brick_dir, int64_t volcount,
+                                    int32_t brick_count)
+{
+        char                    key[PATH_MAX]                   = "";
+        char                    snap_brick_path[PATH_MAX]       = "";
+        char                   *snap_device                     = NULL;
+        gf_boolean_t            add_missed_snap                 = _gf_false;
+        int32_t                 ret                             = -1;
+        xlator_t               *this                            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap_vol);
+        GF_ASSERT (original_brickinfo);
+        GF_ASSERT (snap_brickinfo);
+        GF_ASSERT (snap_brick_dir);
+
+        snprintf (key, sizeof(key) - 1, "vol%ld.brickdir%d", volcount,
+                  brick_count);
+        ret = dict_get_ptr (dict, key, (void **)snap_brick_dir);
+        if (ret) {
+                /* Using original brickinfo here because it will be a
+                 * pending snapshot and storing the original brickinfo
+                 * will help in mapping while recreating the missed snapshot
+                 */
+                gf_log (this->name, GF_LOG_WARNING, "Unable to fetch "
+                        "snap mount path (%s). Using original brickinfo", key);
+                snap_brickinfo->snap_status = -1;
+                strcpy (snap_brick_path, original_brickinfo->path);
+
+                /* In origiator node add snaps missed
+                 * from different nodes to the dict
+                 */
+                if (is_origin_glusterd (dict) == _gf_true)
+                        add_missed_snap = _gf_true;
+        } else {
+                /* Create brick-path in the format /var/run/gluster/snaps/ *
+                 * <snap-uuid>/<original-brick#>/snap-brick-dir *
+                 */
+                snprintf (snap_brick_path, sizeof(snap_brick_path),
+                          "%s/%s/brick%d%s", snap_mount_folder,
+                          snap_vol->volname, brick_count+1,
+                          *snap_brick_dir);
+        }
+
+        if ((snap_brickinfo->snap_status != -1) &&
+            (!uuid_compare (original_brickinfo->uuid, MY_UUID)) &&
+            (!glusterd_is_brick_started (original_brickinfo))) {
+                /* In case if the brick goes down after prevalidate. */
+                gf_log (this->name, GF_LOG_WARNING, "brick %s:%s is not"
+                        " started (snap: %s)",
+                        original_brickinfo->hostname,
+                        original_brickinfo->path,
+                        snap_vol->snapshot->snapname);
+
+                snap_brickinfo->snap_status = -1;
+                strcpy (snap_brick_path, original_brickinfo->path);
+                add_missed_snap = _gf_true;
+        }
+
+        if (add_missed_snap) {
+                ret = glusterd_add_missed_snaps_to_dict (rsp_dict,
+                                            snap_vol->volname,
+                                            original_brickinfo,
+                                            brick_count + 1,
+                                            GF_SNAP_OPTION_TYPE_CREATE);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to add missed"
+                                " snapshot info for %s:%s in the rsp_dict",
+                                original_brickinfo->hostname,
+                                original_brickinfo->path);
+                        goto out;
+                }
+        }
+
+        snprintf (key, sizeof(key), "vol%ld.brick_snapdevice%d",
+                  volcount, brick_count);
+        ret = dict_get_ptr (dict, key, (void **)&snap_device);
+        if (ret) {
+                /* If the device name is empty, so will be the brick path
+                 * Hence the missed snap has already been added above
+                 */
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch "
+                        "snap device (%s). Leaving empty", key);
+        } else
+                strcpy (snap_brickinfo->device_path, snap_device);
+
+        ret = gf_canonicalize_path (snap_brick_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to canonicalize path");
+                goto out;
+        }
+
+        strcpy (snap_brickinfo->hostname, original_brickinfo->hostname);
+        strcpy (snap_brickinfo->path, snap_brick_path);
+        uuid_copy (snap_brickinfo->uuid, original_brickinfo->uuid);
+        list_add_tail (&snap_brickinfo->brick_list, &snap_vol->bricks);
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int32_t
+glusterd_take_brick_snapshot (glusterd_volinfo_t *origin_vol,
+                              glusterd_volinfo_t *snap_vol, dict_t *rsp_dict,
+                              glusterd_brickinfo_t *original_brickinfo,
+                              glusterd_brickinfo_t *snap_brickinfo,
+                              char *snap_brick_dir, int32_t brick_count)
+{
+        char                   *device                          = NULL;
+        int32_t                 ret                             = -1;
+        xlator_t               *this                            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (origin_vol);
+        GF_ASSERT (snap_vol);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (original_brickinfo);
+        GF_ASSERT (snap_brickinfo);
+        GF_ASSERT (snap_brick_dir);
+
+        device = glusterd_take_lvm_snapshot (snap_vol, original_brickinfo);
+        /* Fail the snapshot even though snapshot on one of
+           the bricks fails. At the end when we check whether
+           the snapshot volume meets quorum or not, then the
+           the snapshot can either be treated as success, or
+           in case of failure we can undo the changes and return
+           failure to cli.  */
+        if (!device) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to take snapshot of %s:%s",
+                        original_brickinfo->hostname,
+                        original_brickinfo->path);
+                goto out;
+        }
+
+        /* create the complete brick here */
+        ret = glusterd_snap_brick_create (device, snap_vol,
+                                          original_brickinfo,
+                                          brick_count, snap_brick_dir);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "not able to"
+                        " create the brickinfo for the snap %s"
+                        ", volume %s", snap_vol->snapshot->snapname,
+                        origin_vol->volname);
+                goto out;
+        }
+
+out:
+        if (device)
+                GF_FREE (device);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Look for disconnected peers, for missed snap creates or deletes */
+static int32_t
+glusterd_find_missed_snap (dict_t *rsp_dict, glusterd_volinfo_t *vol,
+                           char *snap_uuid, struct list_head *peers,
+                           int32_t op)
+{
+        int32_t                   brick_count          = -1;
+        int32_t                   ret                  = -1;
+        xlator_t                 *this                 = NULL;
+        glusterd_peerinfo_t      *peerinfo             = NULL;
+        glusterd_brickinfo_t     *brickinfo            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (peers);
+        GF_ASSERT (vol);
+        GF_ASSERT (snap_uuid);
+
+        brick_count = 0;
+        list_for_each_entry (brickinfo, &vol->bricks, brick_list) {
+                if (!uuid_compare (brickinfo->uuid, MY_UUID)) {
+                        /* If the brick belongs to the same node */
+                        brick_count++;
+                        continue;
+                }
+
+                list_for_each_entry (peerinfo, peers, uuid_list) {
+                        if (uuid_compare (peerinfo->uuid, brickinfo->uuid)) {
+                                /* If the brick doesnt belong to this peer */
+                                continue;
+                        }
+
+                        /* Found peer who owns the brick,    *
+                         * if peer is not connected or not   *
+                         * friend add it to missed snap list */
+                        if (!(peerinfo->connected) ||
+                           (peerinfo->state.state !=
+                             GD_FRIEND_STATE_BEFRIENDED)) {
+                                ret = glusterd_add_missed_snaps_to_dict
+                                                   (rsp_dict,
+                                                    snap_uuid,
+                                                    brickinfo,
+                                                    brick_count + 1,
+                                                    op);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to add missed snapshot "
+                                                "info for %s:%s in the "
+                                                "rsp_dict", brickinfo->hostname,
+                                                brickinfo->path);
+                                        goto out;
+                                }
+                        }
+                }
+                brick_count++;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+glusterd_volinfo_t *
+glusterd_do_snap_vol (glusterd_volinfo_t *origin_vol, glusterd_snap_t *snap,
+                      dict_t *dict, dict_t *rsp_dict, int64_t volcount)
+{
+        char                    key[PATH_MAX]                   = "";
+        char                   *snap_brick_dir                  = NULL;
+        char                   *username                        = NULL;
+        char                   *password                        = NULL;
+        glusterd_brickinfo_t   *brickinfo                       = NULL;
+        glusterd_conf_t        *priv                            = NULL;
+        glusterd_volinfo_t     *snap_vol                        = NULL;
+        uuid_t                 *snap_volid                      = NULL;
+        int32_t                 ret                             = -1;
+        int32_t                 brick_count                     = 0;
+        glusterd_brickinfo_t   *snap_brickinfo                  = NULL;
+        xlator_t               *this                            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (origin_vol);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        /* fetch username, password and vol_id from dict*/
+        snprintf (key, sizeof(key), "volume%ld_username", volcount);
+        ret = dict_get_str (dict, key, &username);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get %s for "
+                        "snap %s", key, snap->snapname);
+                goto out;
+        }
+
+        snprintf (key, sizeof(key), "volume%ld_password", volcount);
+        ret = dict_get_str (dict, key, &password);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get %s for "
+                        "snap %s", key, snap->snapname);
+                goto out;
+        }
+        snprintf (key, sizeof(key) - 1, "vol%ld_volid", volcount);
+        ret = dict_get_bin (dict, key, (void **)&snap_volid);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to fetch snap_volid");
+                goto out;
+        }
+
+        /* We are not setting the username and password here as
+         * we need to set the user name and password passed in
+         * the dictionary
+         */
+        ret = glusterd_volinfo_dup (origin_vol, &snap_vol, _gf_false);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to duplicate volinfo "
+                        "for the snapshot %s", snap->snapname);
+                goto out;
+        }
+
+        /* uuid is used as lvm snapshot name.
+           This will avoid restrictions on snapshot names provided by user */
+        GLUSTERD_GET_UUID_NOHYPHEN (snap_vol->volname, *snap_volid);
+        uuid_copy (snap_vol->volume_id, *snap_volid);
+        snap_vol->is_snap_volume = _gf_true;
+        strcpy (snap_vol->parent_volname, origin_vol->volname);
+        snap_vol->snapshot = snap;
+
+        glusterd_auth_set_username (snap_vol, username);
+        glusterd_auth_set_password (snap_vol, password);
+
+        /* Adding snap brickinfos to the snap volinfo */
+        brick_count = 0;
+        list_for_each_entry (brickinfo, &origin_vol->bricks, brick_list) {
+                snap_brickinfo = NULL;
+
+                ret = glusterd_brickinfo_new (&snap_brickinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "initializing the brick for the snap "
+                                "volume failed (snapname: %s)", snap->snapname);
+                        goto out;
+                }
+
+                ret = glusterd_add_bricks_to_snap_volume (dict, rsp_dict,
+                                                          snap_vol,
+                                                          brickinfo,
+                                                          snap_brickinfo,
+                                                          &snap_brick_dir,
+                                                          volcount,
+                                                          brick_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add the snap brick for "
+                                "%s:%s to the snap volume",
+                                brickinfo->hostname, brickinfo->path);
+                        GF_FREE (snap_brickinfo);
+                        goto out;
+                }
+
+                /* Take snapshot of the brick */
+                if ((uuid_compare (brickinfo->uuid, MY_UUID)) ||
+                    (snap_brickinfo->snap_status == -1)) {
+                        brick_count++;
+                        continue;
+                }
+
+                ret = glusterd_take_brick_snapshot (origin_vol, snap_vol,
+                                                    rsp_dict, brickinfo,
+                                                    snap_brickinfo,
+                                                    snap_brick_dir,
+                                                    brick_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to take snapshot for %s:%s",
+                                brickinfo->hostname, brickinfo->path);
+                        goto out;
+                }
+
+                brick_count++;
+        }
+
+        /*TODO: the quorum check of the snap volume here */
+
+        ret = glusterd_store_volinfo (snap_vol,
+                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store snapshot "
+                        "volinfo (%s) for snap %s", snap_vol->volname,
+                        snap->snapname);
+                goto out;
+        }
+
+        ret = generate_brick_volfiles (snap_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "generating the brick "
+                        "volfiles for the snap %s (volume: %s) failed",
+                        snap->snapname, origin_vol->volname);
+                goto out;
+        }
+
+        ret = generate_client_volfiles (snap_vol, GF_CLIENT_TRUSTED);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "generating the trusted "
+                        "client volfiles for the snap %s (volume: %s) failed",
+                        snap->snapname, origin_vol->volname);
+                goto out;
+        }
+        ret = generate_client_volfiles (snap_vol, GF_CLIENT_OTHER);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "generating the client "
+                        "volfiles for the snap %s (volume: %s) failed",
+                        snap->snapname, origin_vol->volname);
+                goto out;
+        }
+
+        ret = glusterd_list_add_snapvol (origin_vol, snap_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "could not add the snap "
+                        "volume %s to the list", snap_vol->volname);
+                goto out;
+        }
+
+        list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                if (brickinfo->snap_status == -1) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "not starting snap brick %s:%s for "
+                                "for the snap %s (volume: %s)",
+                                brickinfo->hostname, brickinfo->path,
+                                snap->snapname, origin_vol->volname);
+                        continue;
+                }
+
+                ret = glusterd_brick_start (snap_vol, brickinfo, _gf_true);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "starting the "
+                                "brick %s:%s for the snap %s (volume: %s) "
+                                "failed", brickinfo->hostname, brickinfo->path,
+                                snap->snapname, origin_vol->volname);
+                        goto out;
+                }
+        }
+
+        snap_vol->status = GLUSTERD_STATUS_STARTED;
+        ret = glusterd_store_volinfo (snap_vol,
+                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to store snap volinfo");
+                goto out;
+        }
+
+out:
+        if (ret) {
+                if (snap_vol)
+                        glusterd_snap_volume_remove (rsp_dict, snap_vol,
+                                                     _gf_true, _gf_true);
+                snap_vol = NULL;
+        }
+
+        return snap_vol;
+}
+
+/* This is a snapshot remove handler function. This function will be
+ * executed in the originator node. This function is responsible for
+ * calling mgmt v3 framework to do the actual remove on all the bricks
+ *
+ * @param req           RPC request object
+ * @param op            gluster operation
+ * @param dict          dictionary containing snapshot remove request
+ * @param err_str       In case of an err this string should be populated
+ * @param len           length of err_str buffer
+ *
+ * @return              Negative value on Failure and 0 in success
+ */
+int
+glusterd_handle_snapshot_remove (rpcsvc_request_t *req, glusterd_op_t op,
+                                 dict_t *dict, char *err_str, size_t len)
+{
+        int                      ret            = -1;
+        int64_t                  volcount       = 0;
+        char                    *snapname       = NULL;
+        char                    *volname        = NULL;
+        char                     key[PATH_MAX]  = "";
+        glusterd_snap_t         *snap           = NULL;
+        glusterd_volinfo_t      *snap_vol       = NULL;
+        glusterd_volinfo_t      *tmp            = NULL;
+        xlator_t                *this           = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        GF_ASSERT (err_str);
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get snapname");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (!snap) {
+                snprintf (err_str, len, "Snap (%s) does not exist", snapname);
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s", err_str);
+                ret = -1;
+                goto out;
+        }
+
+        /* Set volnames in the dict to get mgmt_v3 lock */
+        list_for_each_entry_safe (snap_vol, tmp, &snap->volumes, vol_list) {
+                volcount++;
+                volname = gf_strdup (snap_vol->parent_volname);
+                if (!volname) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR, "strdup failed");
+                        goto out;
+                }
+
+                snprintf (key, sizeof (key), "volname%ld", volcount);
+                ret = dict_set_dynstr (dict, key, volname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                "volume name in dictionary");
+                        GF_FREE (volname);
+                        goto out;
+                }
+                volname = NULL;
+        }
+        ret = dict_set_int64 (dict, "volcount", volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set volcount");
+                goto out;
+        }
+
+        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate snap "
+                        "phases");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_remove_prevalidate (dict_t *dict, char **op_errstr,
+                                      dict_t *rsp_dict)
+{
+        int32_t             ret          = -1;
+        char               *snapname     = NULL;
+        xlator_t           *this        = NULL;
+        glusterd_snap_t    *snap        = NULL;
+
+        this = THIS;
+
+        if (!dict || !op_errstr) {
+                gf_log (this->name, GF_LOG_ERROR, "input parameters NULL");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Getting the snap name "
+                        "failed");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (!snap) {
+                gf_log (this->name, GF_LOG_ERROR, "Snap %s does not exist",
+                        snapname);
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_status_prevalidate (dict_t *dict, char **op_errstr,
+                                      dict_t *rsp_dict)
+{
+        int                     ret             =       -1;
+        char                    *snapname       =       NULL;
+        glusterd_conf_t         *conf           =       NULL;
+        xlator_t                *this           =       NULL;
+        int32_t                 cmd             =       -1;
+        glusterd_volinfo_t      *volinfo        =       NULL;
+        char                    *volname        =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+
+        GF_ASSERT (conf);
+        GF_ASSERT (op_errstr);
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR, "Input dict is NULL");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not fetch status cmd");
+                goto out;
+        }
+
+        switch (cmd) {
+                case GF_SNAP_STATUS_TYPE_ALL:
+                {
+                        break;
+                }
+                case GF_SNAP_STATUS_TYPE_SNAP:
+                {
+                        ret = dict_get_str (dict, "snapname", &snapname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not fetch snapname");
+                                goto out;
+                        }
+
+                        if (!glusterd_find_snap_by_name (snapname)) {
+                                ret = gf_asprintf (op_errstr, "Snap (%s) "
+                                                  "not found", snapname);
+                                if (ret < 0) {
+                                        goto out;
+                                }
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR, "Snap (%s) "
+                                        "not found", snapname);
+                                goto out;
+                        }
+                        break;
+                }
+                case GF_SNAP_STATUS_TYPE_VOL:
+                {
+                        ret = dict_get_str (dict, "volname", &volname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not fetch volname");
+                                goto out;
+                        }
+
+                        ret = glusterd_volinfo_find (volname, &volinfo);
+                        if (ret) {
+                                ret = gf_asprintf (op_errstr, "Volume (%s)"
+                                                  "not found", volname);
+                                if (ret < 0) {
+                                        goto out;
+                                }
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR, "Volume "
+                                        "%s not present", volname);
+                                goto out;
+                        }
+                        break;
+
+                }
+                default:
+                {
+                        gf_log (this->name, GF_LOG_ERROR, "Invalid command");
+                        break;
+                }
+        }
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_snapshot_remove_commit (dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+        int32_t                   ret                  = -1;
+        char                     *snapname             = NULL;
+        char                     *dup_snapname         = NULL;
+        glusterd_snap_t          *snap                 = NULL;
+        glusterd_conf_t          *priv                 = NULL;
+        glusterd_volinfo_t       *snap_volinfo         = NULL;
+        xlator_t                 *this                 = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (op_errstr);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (!dict || !op_errstr) {
+                gf_log (this->name, GF_LOG_ERROR, "input parameters NULL");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Getting the snap name "
+                        "failed");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (!snap) {
+                gf_log (this->name, GF_LOG_ERROR, "Snap %s does not exist",
+                        snapname);
+                ret = -1;
+                goto out;
+        }
+
+        if (is_origin_glusterd (dict) == _gf_true) {
+                /* TODO : As of now there is only volume in snapshot.
+                 * Change this when multiple volume snapshot is introduced
+                 */
+                snap_volinfo = list_entry (snap->volumes.next,
+                                           glusterd_volinfo_t,
+                                           vol_list);
+                if (!snap_volinfo) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to fetch snap_volinfo");
+                        ret = -1;
+                        goto out;
+                }
+
+                /* From origin glusterd check if      *
+                 * any peers with snap bricks is down */
+                ret = glusterd_find_missed_snap (rsp_dict, snap_volinfo,
+                                                 snap_volinfo->volname,
+                                                 &priv->peers,
+                                                 GF_SNAP_OPTION_TYPE_DELETE);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to find missed snap deletes");
+                        goto out;
+                }
+        }
+
+        ret = glusterd_snap_remove (rsp_dict, snap, _gf_true, _gf_false);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to remove snap %s",
+                        snapname);
+                goto out;
+        }
+
+        dup_snapname = gf_strdup (snapname);
+        if (!dup_snapname) {
+                gf_log (this->name, GF_LOG_ERROR, "Strdup failed");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (rsp_dict, "snapname", dup_snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set the snapname");
+                GF_FREE (dup_snapname);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_do_snap_cleanup (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t                  ret                   = -1;
+        char                     *name                 = NULL;
+        xlator_t                 *this                 = NULL;
+        glusterd_conf_t          *conf                 = NULL;
+        glusterd_volinfo_t       *volinfo              = NULL;
+        glusterd_snap_t          *snap                 = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        if (!dict || !op_errstr) {
+                gf_log (this->name, GF_LOG_ERROR, "input parameters NULL");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &name);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "getting the snap "
+                        "name failed (volume: %s)", volinfo->volname);
+                goto out;
+        }
+
+        /*
+          If the snapname is not found that means the failure happened at
+          staging, or in commit, before the snap object is created, in which
+          case there is nothing to cleanup. So set ret to 0.
+        */
+        snap = glusterd_find_snap_by_name (name);
+        if (!snap) {
+                gf_log (this->name, GF_LOG_INFO, "snap %s is not found", name);
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_snap_remove (rsp_dict, snap, _gf_true, _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "removing the snap %s failed",
+                        name);
+                goto out;
+        }
+
+        name = NULL;
+
+        ret = 0;
+
+out:
+
+        return ret;
+}
+
+/* In case of a successful, delete or create operation, during post_validate *
+ * look for missed snap operations and update the missed snap lists */
+int32_t
+glusterd_snapshot_update_snaps_post_validate (dict_t *dict, char **op_errstr,
+                                              dict_t *rsp_dict)
+{
+        int32_t                  ret                    = -1;
+        int32_t                  missed_snap_count      = -1;
+        xlator_t                *this                   = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (op_errstr);
+
+        ret = dict_get_int32 (dict, "missed_snap_count",
+                              &missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "No missed snaps");
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_store_update_missed_snaps (dict, missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to update missed_snaps_list");
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_snapshot_create_commit (dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+        int                     ret                     = -1;
+        int64_t                 i                       = 0;
+        int64_t                 volcount                = 0;
+        char                    *snapname               = NULL;
+        char                    *volname                = NULL;
+        char                    *tmp_name               = NULL;
+        char                    key[PATH_MAX]           = "";
+        xlator_t                *this                   = NULL;
+        glusterd_snap_t         *snap                   = NULL;
+        glusterd_volinfo_t      *origin_vol             = NULL;
+        glusterd_volinfo_t      *snap_vol               = NULL;
+        glusterd_conf_t *priv   = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(dict);
+        GF_ASSERT(op_errstr);
+        GF_ASSERT(rsp_dict);
+        priv = this->private;
+        GF_ASSERT(priv);
+
+        ret = dict_get_int64 (dict, "volcount", &volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                        "get the volume count");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snapname");
+                goto out;
+        }
+        tmp_name = gf_strdup (snapname);
+        if (!tmp_name) {
+                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (rsp_dict, "snapname", tmp_name);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set snapname in rsp_dict");
+                GF_FREE (tmp_name);
+                goto out;
+        }
+        tmp_name = NULL;
+
+        snap = glusterd_create_snap_object (dict, rsp_dict);
+        if (!snap) {
+                gf_log (this->name, GF_LOG_ERROR, "creating the"
+                        "snap object %s failed", snapname);
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 1; i <= volcount; i++) {
+                snprintf (key, sizeof (key), "volname%ld", i);
+                ret = dict_get_str (dict, key, &volname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to get volume name");
+                        goto out;
+                }
+
+                ret = glusterd_volinfo_find (volname, &origin_vol);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to get the volinfo for "
+                                "the volume %s", volname);
+                        goto out;
+                }
+
+                /* TODO: Create a stub where the bricks are
+                   added parallely by worker threads so that
+                   the snap creating happens parallely.  */
+                snap_vol = glusterd_do_snap_vol (origin_vol, snap, dict,
+                                                 rsp_dict, i);
+                if (!snap_vol) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_WARNING, "taking the "
+                                "snapshot of the volume %s failed", volname);
+                        goto out;
+                }
+        }
+
+        snap->snap_status = GD_SNAP_STATUS_IN_USE;
+        ret = glusterd_store_snap (snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Could not store snap"
+                        "object %s", snap->snapname);
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        if (ret) {
+                if (snap)
+                        glusterd_snap_remove (rsp_dict, snap,
+                                              _gf_true, _gf_true);
+                snap = NULL;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+snap_max_hard_limit_set_commit (dict_t *dict, uint64_t value,
+                                char *volname, char **op_errstr)
+{
+        char                err_str[PATH_MAX]    = "";
+        glusterd_conf_t    *conf                 = NULL;
+        glusterd_volinfo_t *volinfo              = NULL;
+        int                 ret                  = -1;
+        xlator_t           *this                 = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (volname);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        /* TODO: Initiate auto deletion when there is a limit change */
+        if (!volname) {
+                /* For system limit */
+                conf->snap_max_hard_limit = value;
+
+                ret = glusterd_store_global_info (this);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX, "Failed to store "
+                                  "snap-max-hard-limit for system");
+                        goto out;
+                }
+        } else {
+               /*  For one volume */
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX, "Failed to get the"
+                                   " volinfo for volume %s", volname);
+                        goto out;
+                }
+
+                volinfo->snap_max_hard_limit = value;
+
+                ret = glusterd_store_volinfo (volinfo,
+                                        GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX, "Failed to store "
+                                 "snap-max-hard-limit for volume %s", volname);
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        if (ret) {
+                *op_errstr = gf_strdup (err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+        }
+        return ret;
+}
+
+int
+snap_max_limits_display_commit (dict_t *rsp_dict, char *volname,
+                                char **op_errstr)
+{
+        char                err_str[PATH_MAX]    = "";
+        char                buf[PATH_MAX]        = "";
+        glusterd_conf_t    *conf                 = NULL;
+        glusterd_volinfo_t *volinfo              = NULL;
+        int                 ret                  = -1;
+        uint64_t            active_hard_limit    = 0;
+        uint64_t            snap_max_limit       = 0;
+        uint64_t            soft_limit_value     = -1;
+        uint64_t            count                = 0;
+        xlator_t           *this                 = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (volname);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        if (!volname) {
+                /* For system limit */
+                list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                        if (volinfo->is_snap_volume == _gf_true)
+                                continue;
+                        snap_max_limit = volinfo->snap_max_hard_limit;
+                        if (snap_max_limit > conf->snap_max_hard_limit)
+                                active_hard_limit = conf->snap_max_hard_limit;
+                        else
+                                active_hard_limit = snap_max_limit;
+                        soft_limit_value = (active_hard_limit *
+                                            conf->snap_max_soft_limit) / 100;
+
+                        snprintf (buf, sizeof(buf), "volume%ld-volname", count);
+                        ret = dict_set_str (rsp_dict, buf, volinfo->volname);
+                        if (ret) {
+                                snprintf (err_str, PATH_MAX,
+                                          "Failed to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-snap-max-hard-limit", count);
+                        ret = dict_set_uint64 (rsp_dict, buf, snap_max_limit);
+                        if (ret) {
+                                snprintf (err_str, PATH_MAX,
+                                          "Failed to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-active-hard-limit", count);
+                        ret = dict_set_uint64 (rsp_dict, buf,
+                                               active_hard_limit);
+                        if (ret) {
+                                snprintf (err_str, PATH_MAX,
+                                          "Failed to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-snap-max-soft-limit", count);
+                        ret = dict_set_uint64 (rsp_dict, buf, soft_limit_value);
+                        if (ret) {
+                                snprintf (err_str, PATH_MAX,
+                                          "Failed to set %s", buf);
+                                goto out;
+                        }
+                        count++;
+                }
+
+                ret = dict_set_uint64 (rsp_dict, "voldisplaycount", count);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set voldisplaycount");
+                        goto out;
+                }
+        } else {
+                /*  For one volume */
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX, "Failed to get the"
+                                   " volinfo for volume %s", volname);
+                        goto out;
+                }
+
+                snap_max_limit = volinfo->snap_max_hard_limit;
+                if (snap_max_limit > conf->snap_max_hard_limit)
+                        active_hard_limit = conf->snap_max_hard_limit;
+                else
+                        active_hard_limit = snap_max_limit;
+
+                soft_limit_value = (active_hard_limit *
+                                    conf->snap_max_soft_limit) / 100;
+
+                snprintf (buf, sizeof(buf), "volume%ld-volname", count);
+                ret = dict_set_str (rsp_dict, buf, volinfo->volname);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set %s", buf);
+                        goto out;
+                }
+
+                snprintf (buf, sizeof(buf),
+                          "volume%ld-snap-max-hard-limit", count);
+                ret = dict_set_uint64 (rsp_dict, buf, snap_max_limit);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set %s", buf);
+                        goto out;
+                }
+
+                snprintf (buf, sizeof(buf),
+                          "volume%ld-active-hard-limit", count);
+                ret = dict_set_uint64 (rsp_dict, buf, active_hard_limit);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set %s", buf);
+                        goto out;
+                }
+
+                snprintf (buf, sizeof(buf),
+                          "volume%ld-snap-max-soft-limit", count);
+                ret = dict_set_uint64 (rsp_dict, buf, soft_limit_value);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set %s", buf);
+                        goto out;
+                }
+
+                count++;
+
+                ret = dict_set_uint64 (rsp_dict, "voldisplaycount", count);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set voldisplaycount");
+                        goto out;
+                }
+
+        }
+
+        ret = dict_set_uint64 (rsp_dict, "snap-max-hard-limit",
+                               conf->snap_max_hard_limit);
+        if (ret) {
+                snprintf (err_str, PATH_MAX,
+                          "Failed to set sys-snap-max-hard-limit ");
+                goto out;
+        }
+
+        ret = dict_set_uint64 (rsp_dict, "snap-max-soft-limit",
+                               conf->snap_max_soft_limit);
+        if (ret) {
+                snprintf (err_str, PATH_MAX,
+                          "Failed to set sys-snap-max-hard-limit ");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (ret) {
+                *op_errstr = gf_strdup (err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+        }
+        return ret;
+}
+
+int
+glusterd_snapshot_config_commit (dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+        char               *volname              = NULL;
+        xlator_t           *this                 = NULL;
+        int                 ret                  = -1;
+        char                err_str[PATH_MAX]    = {0,};
+        glusterd_conf_t    *conf                 = NULL;
+        int                 config_command       = 0;
+        uint64_t            hard_limit           = 0;
+        uint64_t            soft_limit           = 0;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        ret = dict_get_int32 (dict, "config-command", &config_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get config-command type");
+                goto out;
+        }
+
+        /* Ignore the return value of the following dict_get,
+         * as they are optional
+         */
+        ret = dict_get_str (dict, "volname", &volname);
+
+        ret = dict_get_uint64 (dict, "snap-max-hard-limit", &hard_limit);
+
+        ret = dict_get_uint64 (dict, "snap-max-soft-limit", &soft_limit);
+
+        switch (config_command) {
+        case GF_SNAP_CONFIG_TYPE_SET:
+                if (hard_limit) {
+                        /* Commit ops for snap-max-hard-limit */
+                        ret = snap_max_hard_limit_set_commit (dict, hard_limit,
+                                                              volname,
+                                                              op_errstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "snap-max-hard-limit set "
+                                        "commit failed.");
+                                goto out;
+                        }
+                }
+
+                if (soft_limit) {
+                        /* For system limit */
+                        conf->snap_max_soft_limit = soft_limit;
+
+                        ret = glusterd_store_global_info (this);
+                        if (ret) {
+                                snprintf (err_str, PATH_MAX, "Failed to store "
+                                          "snap-max-soft-limit for system");
+                                *op_errstr = gf_strdup (err_str);
+                                gf_log (this->name, GF_LOG_ERROR, "%s",
+                                        err_str);
+                                goto out;
+                        }
+                }
+                break;
+
+        case GF_SNAP_CONFIG_DISPLAY:
+                /* Reading data from local node only */
+                if (!is_origin_glusterd (dict)) {
+                        ret = 0;
+                        break;
+                }
+
+                ret = snap_max_limits_display_commit (rsp_dict, volname,
+                                                      op_errstr);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "snap-max-limit "
+                                "display commit failed.");
+                        goto out;
+                }
+                break;
+        default:
+                break;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_get_brick_lvm_details (dict_t *rsp_dict,
+                               glusterd_brickinfo_t *brickinfo, char *volname,
+                                char *device, char *key_prefix)
+{
+
+        int                     ret             =       -1;
+        glusterd_conf_t         *priv           =       NULL;
+        runner_t                runner          =       {0,};
+        xlator_t                *this           =       NULL;
+        char                    msg[PATH_MAX]   =       "";
+        char                    buf[PATH_MAX]   =       "";
+        char                    *ptr            =       NULL;
+        char                    *token          =       NULL;
+        char                    key[PATH_MAX]   =       "";
+        char                    *value          =       NULL;
+
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (brickinfo);
+        GF_ASSERT (volname);
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        device = glusterd_get_brick_mount_details (brickinfo);
+        if (!device) {
+                gf_log (this->name, GF_LOG_ERROR, "Getting device name for "
+                        "the brick %s:%s failed", brickinfo->hostname,
+                         brickinfo->path);
+                goto out;
+        }
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "running lvs command, "
+                  "for getting snap status");
+        /* Using lvs command fetch the Volume Group name,
+         * Percentage of data filled and Logical Volume size
+         *
+         * "-o" argument is used to get the desired information,
+         * example : "lvs /dev/VolGroup/thin_vol -o vgname,lv_size",
+         * will get us Volume Group name and Logical Volume size.
+         *
+         * Here separator used is ":",
+         * for the above given command with separator ":",
+         * The output will be "vgname:lvsize"
+         */
+        runner_add_args (&runner, "lvs", device, "--noheading", "-o",
+                         "vg_name,data_percent,lv_size",
+                         "--separator", ":", NULL);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+        ret = runner_start (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not perform lvs action");
+                goto end;
+        }
+        do {
+                ptr = fgets (buf, sizeof (buf),
+                             runner_chio (&runner, STDOUT_FILENO));
+
+                if (ptr == NULL)
+                        break;
+                token = strtok (buf, ":");
+                if (token != NULL) {
+                        while (token && token[0] == ' ')
+                                token++;
+                        if (!token) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Invalid vg entry");
+                                goto end;
+                        }
+                        value = gf_strdup (token);
+                        if (!value) {
+                                ret = -1;
+                                goto end;
+                        }
+                        ret = snprintf (key, sizeof (key), "%s.vgname",
+                                        key_prefix);
+                        if (ret < 0) {
+                                goto end;
+                        }
+
+                        ret = dict_set_dynstr (rsp_dict, key, value);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not save vgname ");
+                                goto end;
+                        }
+                }
+
+                token = strtok (NULL, ":");
+                if (token != NULL) {
+                        value = gf_strdup (token);
+                        if (!value) {
+                                ret = -1;
+                                goto end;
+                        }
+                        ret = snprintf (key, sizeof (key), "%s.data",
+                                        key_prefix);
+                        if (ret < 0) {
+                                goto end;
+                        }
+
+                        ret = dict_set_dynstr (rsp_dict, key, value);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not save data percent ");
+                                goto end;
+                        }
+                }
+                token = strtok (NULL, ":");
+                if (token != NULL) {
+                        value = gf_strdup (token);
+                        if (!value) {
+                                ret = -1;
+                                goto end;
+                        }
+                        ret = snprintf (key, sizeof (key), "%s.lvsize",
+                                        key_prefix);
+                        if (ret < 0) {
+                                goto end;
+                        }
+
+                        ret = dict_set_dynstr (rsp_dict, key, value);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not save meta data percent ");
+                                goto end;
+                        }
+                }
+
+        } while (ptr != NULL);
+
+        ret = 0;
+
+end:
+        runner_end (&runner);
+
+out:
+        if (ret && value) {
+                GF_FREE (value);
+        }
+
+        return ret;
+}
+
+int
+glusterd_get_single_brick_status (char **op_errstr, dict_t *rsp_dict,
+                                 char *keyprefix, int index,
+                                 glusterd_volinfo_t *snap_volinfo,
+                                 glusterd_brickinfo_t *brickinfo)
+{
+        int             ret                     = -1;
+        xlator_t        *this                   = NULL;
+        glusterd_conf_t *priv                   = NULL;
+        char            key[PATH_MAX]           = "";
+        char            *device                 = NULL;
+        char            *value                  = NULL;
+        char            brick_path[PATH_MAX]    = "";
+        char            pidfile[PATH_MAX]       = "";
+        pid_t           pid                     = -1;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (keyprefix);
+        GF_ASSERT (snap_volinfo);
+        GF_ASSERT (brickinfo);
+
+        ret = snprintf (key, sizeof (key), "%s.brick%d.path", keyprefix,
+                        index);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = snprintf (brick_path, sizeof (brick_path),
+                        "%s:%s", brickinfo->hostname, brickinfo->path);
+        if (ret < 0) {
+                goto out;
+        }
+
+        value = gf_strdup (brick_path);
+        if (!value) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (rsp_dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to store "
+                        "brick_path %s", brickinfo->path);
+                goto out;
+        }
+
+        if (brickinfo->snap_status == -1) {
+                /* Setting vgname as "Pending Snapshot" */
+                value = gf_strdup ("Pending Snapshot");
+                if (!value) {
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (key, sizeof (key), "%s.brick%d.vgname",
+                          keyprefix, index);
+                ret = dict_set_dynstr (rsp_dict, key, value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not save vgname ");
+                        goto out;
+                }
+
+                ret = 0;
+                goto out;
+        }
+        value = NULL;
+
+        ret = snprintf (key, sizeof (key), "%s.brick%d.status",
+                        keyprefix, index);
+        if (ret < 0) {
+                goto out;
+        }
+
+        if (brickinfo->status == GF_BRICK_STOPPED) {
+                value = gf_strdup ("No");
+                if (!value) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_str (rsp_dict, key, value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not save brick status");
+                        goto out;
+                }
+                value = NULL;
+        } else {
+                value = gf_strdup ("Yes");
+                if (!value) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_str (rsp_dict, key, value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not save brick status");
+                        goto out;
+                }
+                value = NULL;
+
+                GLUSTERD_GET_BRICK_PIDFILE (pidfile, snap_volinfo,
+                                            brickinfo, priv);
+                ret = glusterd_is_service_running (pidfile, &pid);
+
+                ret = snprintf (key, sizeof (key), "%s.brick%d.pid",
+                                keyprefix, index);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_set_int32 (rsp_dict, key, pid);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not save pid %d", pid);
+                        goto out;
+                }
+        }
+
+        ret = snprintf (key, sizeof (key), "%s.brick%d",
+                        keyprefix, index);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = glusterd_get_brick_lvm_details (rsp_dict, brickinfo,
+                                              snap_volinfo->volname,
+                                              device, key);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                        "brick LVM details");
+                goto out;
+        }
+out:
+        if (ret && value) {
+                GF_FREE (value);
+        }
+
+        return ret;
+}
+
+int
+glusterd_get_single_snap_status (char **op_errstr, dict_t *rsp_dict,
+                                 char *keyprefix, glusterd_snap_t *snap)
+{
+        int                      ret                 =       -1;
+        xlator_t                *this                =       NULL;
+        char                     key[PATH_MAX]       =       "";
+        char                     brickkey[PATH_MAX]  =       "";
+        glusterd_volinfo_t      *snap_volinfo        =       NULL;
+        glusterd_volinfo_t      *tmp_volinfo         =       NULL;
+        glusterd_brickinfo_t    *brickinfo           =       NULL;
+        int                      volcount            =       0;
+        int                      brickcount          =       0;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (keyprefix);
+        GF_ASSERT (snap);
+
+        list_for_each_entry_safe (snap_volinfo, tmp_volinfo, &snap->volumes,
+                                  vol_list) {
+                ret = snprintf (key, sizeof (key), "%s.vol%d", keyprefix,
+                                volcount);
+                if (ret < 0) {
+                        goto out;
+                }
+                list_for_each_entry (brickinfo, &snap_volinfo->bricks,
+                                     brick_list) {
+                        if (!glusterd_is_local_brick (this, snap_volinfo,
+                            brickinfo)) {
+                                brickcount++;
+                                continue;
+                        }
+
+                        ret = glusterd_get_single_brick_status (op_errstr,
+                                                  rsp_dict, key, brickcount,
+                                                  snap_volinfo, brickinfo);
+
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Getting "
+                                        "single snap status failed");
+                                goto out;
+                        }
+                        brickcount++;
+                }
+                ret = snprintf (brickkey, sizeof (brickkey), "%s.brickcount",
+                                key);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_set_int32 (rsp_dict, brickkey, brickcount);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not save brick count");
+                        goto out;
+                }
+                volcount++;
+        }
+
+        ret = snprintf (key, sizeof (key), "%s.volcount", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = dict_set_int32 (rsp_dict, key, volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not save volcount");
+                goto out;
+        }
+
+out:
+
+        return ret;
+}
+
+int
+glusterd_get_each_snap_object_status (char **op_errstr, dict_t *rsp_dict,
+                                      glusterd_snap_t *snap, char *keyprefix)
+{
+        int                     ret             =       -1;
+        char                    key[PATH_MAX]   =       "";
+        char                    *temp           =       NULL;
+        xlator_t                *this           =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap);
+        GF_ASSERT (keyprefix);
+
+        /* TODO : Get all the snap volume info present in snap object,
+         * as of now, There will be only one snapvolinfo per snap object
+         */
+        ret = snprintf (key, sizeof (key), "%s.snapname", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        temp = gf_strdup (snap->snapname);
+        if (temp == NULL) {
+                ret = -1;
+                goto out;
+        }
+        ret = dict_set_dynstr (rsp_dict, key, temp);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not save "
+                        "snap name");
+                goto out;
+        }
+
+        temp = NULL;
+
+        ret = snprintf (key, sizeof (key), "%s.uuid", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        temp = gf_strdup (uuid_utoa (snap->snap_id));
+        if (temp == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (rsp_dict, key, temp);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not save "
+                        "snap UUID");
+                goto out;
+        }
+
+        temp = NULL;
+
+        ret = glusterd_get_single_snap_status (op_errstr, rsp_dict, keyprefix,
+                                               snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not get single snap status");
+                goto out;
+        }
+
+        ret = snprintf (key, sizeof (key), "%s.volcount", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = dict_set_int32 (rsp_dict, key, 1);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not save volcount");
+                goto out;
+        }
+out:
+        if (ret && temp)
+                GF_FREE (temp);
+
+        return ret;
+}
+
+int
+glusterd_get_snap_status_of_volume (char **op_errstr, dict_t *rsp_dict,
+                                    char *volname, char *keyprefix) {
+        int                     ret              =       -1;
+        glusterd_volinfo_t      *snap_volinfo    =       NULL;
+        glusterd_volinfo_t      *temp_volinfo    =       NULL;
+        glusterd_volinfo_t      *volinfo         =       NULL;
+        char                    key[PATH_MAX]    =        "";
+        xlator_t                *this            =       NULL;
+        glusterd_conf_t         *priv            =       NULL;
+        int                     i                =        0;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (volname);
+        GF_ASSERT (keyprefix);
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volinfo of "
+                        "volume %s", volname);
+                goto out;
+        }
+
+        list_for_each_entry_safe (snap_volinfo, temp_volinfo,
+                             &volinfo->snap_volumes, snapvol_list) {
+                ret = snprintf (key, sizeof (key), "status.snap%d", i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = glusterd_get_each_snap_object_status (op_errstr,
+                                       rsp_dict, snap_volinfo->snapshot, key);
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Function : "
+                                "glusterd_get_single_snap_status failed");
+                        goto out;
+                }
+                i++;
+        }
+
+        ret = dict_set_int32 (rsp_dict, "status.snapcount", i);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to save snapcount");
+                ret = -1;
+                goto out;
+        }
+out:
+        return ret;
+}
+
+int
+glusterd_get_all_snapshot_status (dict_t *dict, char **op_errstr,
+                              dict_t *rsp_dict)
+{
+        int32_t                 i               =       0;
+        int                     ret             =       -1;
+        char                    key[PATH_MAX]   =       "";
+        glusterd_conf_t         *priv           =       NULL;
+        glusterd_snap_t         *snap           =       NULL;
+        glusterd_snap_t         *tmp_snap       =       NULL;
+        xlator_t                *this           =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        list_for_each_entry_safe (snap, tmp_snap,
+                                  &priv->snapshots, snap_list) {
+                ret = snprintf (key, sizeof (key), "status.snap%d", i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = glusterd_get_each_snap_object_status (op_errstr,
+                                                         rsp_dict, snap, key);
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Could not get "
+                                "the details of a snap object: %s",
+                                snap->snapname);
+                        goto out;
+                }
+                i++;
+        }
+
+        ret = dict_set_int32 (rsp_dict, "status.snapcount", i);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not save snapcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int
+glusterd_snapshot_status_commit (dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+        xlator_t        *this             =       NULL;
+        int             ret               =       -1;
+        glusterd_conf_t *conf             =       NULL;
+        char            *get_buffer       =       NULL;
+        int32_t         cmd               =       -1;
+        char            *snapname         =       NULL;
+        glusterd_snap_t *snap             =       NULL;
+        char            *volname          =       NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+        ret = dict_get_int32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get status cmd type");
+                goto out;
+        }
+
+        ret = dict_set_int32 (rsp_dict, "cmd", cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not save status cmd in rsp dictionary");
+                goto out;
+        }
+        switch (cmd) {
+                case GF_SNAP_STATUS_TYPE_ALL:
+                {
+                        ret = glusterd_get_all_snapshot_status (dict, op_errstr,
+                                                           rsp_dict);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "get snapshot status");
+                                goto out;
+                        }
+                        break;
+                }
+                case GF_SNAP_STATUS_TYPE_SNAP:
+                {
+
+                        ret = dict_get_str (dict, "snapname", &snapname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "get snap name");
+                                goto out;
+                        }
+
+                        snap = glusterd_find_snap_by_name (snapname);
+                        if (!snap) {
+                                ret = gf_asprintf (op_errstr, "Snap (%s) "
+                                                  "not found", snapname);
+                                if (ret < 0) {
+                                        goto out;
+                                }
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "get snap volinfo");
+                                goto out;
+                        }
+                        ret = glusterd_get_each_snap_object_status (op_errstr,
+                                          rsp_dict, snap, "status.snap0");
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "get status of snap %s", get_buffer);
+                                goto out;
+                        }
+                        break;
+                }
+                case GF_SNAP_STATUS_TYPE_VOL:
+                {
+                        ret = dict_get_str (dict, "volname", &volname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to"
+                                        " get volume name");
+                                goto out;
+                        }
+
+                        ret = glusterd_get_snap_status_of_volume (op_errstr,
+                                           rsp_dict, volname, "status.vol0");
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Function :"
+                                        " glusterd_get_snap_status_of_volume "
+                                        "failed");
+                                goto out;
+                        }
+                }
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_snapshot_create_postvalidate (dict_t *dict, int32_t op_ret,
+                                       char **op_errstr, dict_t *rsp_dict)
+{
+        xlator_t        *this           = NULL;
+        glusterd_conf_t *priv           = NULL;
+        int              ret            = -1;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (op_ret) {
+                ret = glusterd_do_snap_cleanup (dict, op_errstr, rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "cleanup operation "
+                                "failed");
+                        goto out;
+                }
+        } else {
+                ret = glusterd_snapshot_update_snaps_post_validate (dict,
+                                                                    op_errstr,
+                                                                    rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "create snapshot");
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_snapshot (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+
+        xlator_t        *this           = NULL;
+        glusterd_conf_t *priv           = NULL;
+        int32_t          snap_command   = 0;
+        int              ret            = -1;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_int32 (dict, "type", &snap_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case (GF_SNAP_OPTION_TYPE_CREATE):
+                ret = glusterd_snapshot_create_commit (dict, op_errstr,
+                                                       rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "create snapshot");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+                ret = glusterd_snapshot_config_commit (dict, op_errstr,
+                                                       rsp_dict);
+                break;
+
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                ret = glusterd_snapshot_remove_commit (dict, op_errstr,
+                                                       rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "delete snapshot");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+                ret = glusterd_snapshot_restore (dict, op_errstr,
+                                                 rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Failed to "
+                                "restore snapshot");
+                        goto out;
+                }
+
+                break;
+
+        case GF_SNAP_OPTION_TYPE_STATUS:
+                ret = glusterd_snapshot_status_commit (dict, op_errstr,
+                                                       rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "show snapshot status");
+                        goto out;
+                }
+                break;
+
+
+        default:
+                gf_log (this->name, GF_LOG_WARNING, "invalid snap command");
+                goto out;
+                break;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_brickop (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        int            ret       = -1;
+        int64_t        vol_count = 0;
+        int64_t        count     = 1;
+        char           key[1024] = {0,};
+        char           *volname  = NULL;
+        int32_t        snap_command = 0;
+        xlator_t       *this     = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        ret = dict_get_int32 (dict, "type", &snap_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+                ret = dict_get_int64 (dict, "volcount", &vol_count);
+                if (ret)
+                        goto out;
+                while (count <= vol_count) {
+                        snprintf (key, 1024, "volname%"PRId64, count);
+                        ret = dict_get_str (dict, key, &volname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to get volname");
+                                goto out;
+                        }
+                        ret = dict_set_str (dict, "volname", volname);
+                        if (ret)
+                                goto out;
+
+                        ret = gd_brick_op_phase (GD_OP_SNAP, NULL, dict,
+                                        op_errstr);
+                        if (ret)
+                                goto out;
+                        volname = NULL;
+                        count++;
+                }
+
+                dict_del (dict, "volname");
+                ret = 0;
+                break;
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                break;
+        default:
+                break;
+        }
+
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_prevalidate (dict_t *dict, char **op_errstr,
+                               dict_t *rsp_dict)
+{
+        int                snap_command          = 0;
+        xlator_t           *this                 = NULL;
+        int                ret                   = -1;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        ret = dict_get_int32 (dict, "type", &snap_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case (GF_SNAP_OPTION_TYPE_CREATE):
+                ret = glusterd_snapshot_create_prevalidate (dict, op_errstr,
+                                                            rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot create "
+                                "pre-validation failed");
+                        goto out;
+                }
+                break;
+
+        case (GF_SNAP_OPTION_TYPE_CONFIG):
+                ret = glusterd_snapshot_config_prevalidate (dict, op_errstr);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot config "
+                                "pre-validation failed");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+                ret = glusterd_snapshot_restore_prevalidate (dict, op_errstr,
+                                                             rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot restore "
+                                        "validation failed");
+                        goto out;
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                ret = glusterd_snapshot_remove_prevalidate (dict, op_errstr,
+                                                            rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot remove "
+                                "validation failed");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_STATUS:
+                ret = glusterd_snapshot_status_prevalidate (dict, op_errstr,
+                                                            rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot status "
+                                "validation failed");
+                        goto out;
+                }
+                break;
+
+        default:
+                gf_log (this->name, GF_LOG_WARNING, "invalid snap command");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_postvalidate (dict_t *dict, int32_t op_ret, char **op_errstr,
+                                dict_t *rsp_dict)
+{
+        int                snap_command          = 0;
+        xlator_t           *this                 = NULL;
+        int                ret                   = -1;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        ret = dict_get_int32 (dict, "type", &snap_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+                ret = glusterd_snapshot_create_postvalidate (dict, op_ret,
+                                                             op_errstr,
+                                                             rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot create "
+                                "post-validation failed");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_DELETE:
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+                ret = glusterd_snapshot_update_snaps_post_validate (dict,
+                                                                    op_errstr,
+                                                                    rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "update missed snaps list");
+                        goto out;
+                }
+                break;
+
+        default:
+                gf_log (this->name, GF_LOG_WARNING, "invalid snap command");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_handle_snapshot_fn (rpcsvc_request_t *req)
+{
+        int32_t               ret            = 0;
+        dict_t               *dict           = NULL;
+        gf_cli_req            cli_req        = {{0},};
+        glusterd_op_t         cli_op         = GD_OP_SNAP;
+        int                   type           = 0;
+        glusterd_conf_t      *conf           = NULL;
+        char                 *host_uuid      = NULL;
+        char                  err_str[2048]  = {0,};
+        xlator_t             *this           = NULL;
+        char                 *volname        = NULL;
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len > 0) {
+                dict = dict_new ();
+                if (!dict)
+                        goto out;
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+                }
+
+                dict->extra_stdfree = cli_req.dict.dict_val;
+
+                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
+                if (host_uuid == NULL) {
+                        snprintf (err_str, sizeof (err_str), "Failed to get "
+                                  "the uuid of local glusterd");
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
+                if (ret) {
+                        GF_FREE (host_uuid);
+                        goto out;
+                }
+
+
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "request dict length is %d",
+                        cli_req.dict.dict_len);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "type", &type);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Command type not found");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        switch (type) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+                ret = glusterd_handle_snapshot_create (req, cli_op, dict,
+                                                err_str, sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot create "
+                                        "failed: %s", err_str);
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+                ret = glusterd_handle_snapshot_restore (req, cli_op, dict,
+                                                err_str, sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot restore "
+                                        "failed: %s", err_str);
+                }
+
+                break;
+        case GF_SNAP_OPTION_TYPE_INFO:
+                ret = glusterd_handle_snapshot_info (req, cli_op, dict,
+                                                err_str, sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot info "
+                                        "failed");
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_LIST:
+                ret = glusterd_handle_snapshot_list (req, cli_op, dict,
+                                                err_str, sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot list "
+                                        "failed");
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+                /* TODO : Type of lock to be taken when we are setting
+                 * limits system wide
+                 */
+                ret = dict_get_str (dict, "volname", &volname);
+                if (!volname) {
+                        ret = dict_set_int32 (dict, "hold_vol_locks",
+                                              _gf_false);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Unable to set hold_vol_locks value "
+                                        "as _gf_false");
+                                goto out;
+                        }
+
+                }
+                ret = glusterd_mgmt_v3_initiate_all_phases (req, cli_op, dict);
+                break;
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                ret = glusterd_handle_snapshot_remove (req, cli_op, dict,
+                                                       err_str,
+                                                       sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot delete "
+                                "failed: %s", err_str);
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_START:
+        case GF_SNAP_OPTION_TYPE_STOP:
+        case GF_SNAP_OPTION_TYPE_STATUS:
+                ret = glusterd_handle_snapshot_status (req, cli_op, dict,
+                                                       err_str,
+                                                       sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot status "
+                                "failed: %s", err_str);
+                }
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR, "Unkown snapshot request "
+                        "type (%d)", type);
+                ret = -1; /* Failure */
+        }
+
+out:
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+
+        return ret;
+}
+
+int
+glusterd_handle_snapshot (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, glusterd_handle_snapshot_fn);
+}
+
+static inline void
+glusterd_free_snap_op (glusterd_snap_op_t *snap_op)
+{
+        if (snap_op) {
+                if (snap_op->brick_path)
+                        GF_FREE (snap_op->brick_path);
+
+                GF_FREE (snap_op);
+        }
+}
+
+/* Look for duplicates and accordingly update the list */
+int32_t
+glusterd_update_missed_snap_entry (glusterd_missed_snap_info *missed_snapinfo,
+                                   glusterd_snap_op_t *missed_snap_op)
+{
+        int32_t                        ret                         = -1;
+        glusterd_snap_op_t            *snap_opinfo                 = NULL;
+        gf_boolean_t                   match                       = _gf_false;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(missed_snapinfo);
+        GF_ASSERT(missed_snap_op);
+
+        list_for_each_entry (snap_opinfo, &missed_snapinfo->snap_ops,
+                             snap_ops_list) {
+                if ((!strcmp (snap_opinfo->brick_path,
+                              missed_snap_op->brick_path)) &&
+                    (snap_opinfo->op == missed_snap_op->op)) {
+                        /* If two entries have conflicting status
+                         * GD_MISSED_SNAP_DONE takes precedence
+                         */
+                        if ((snap_opinfo->status == GD_MISSED_SNAP_PENDING) &&
+                            (missed_snap_op->status == GD_MISSED_SNAP_DONE)) {
+                                snap_opinfo->status = GD_MISSED_SNAP_DONE;
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "Updating missed snap status "
+                                        "for %s:%d:%s:%d as DONE",
+                                        missed_snapinfo->node_snap_info,
+                                        snap_opinfo->brick_num,
+                                        snap_opinfo->brick_path,
+                                        snap_opinfo->op);
+                                ret = 0;
+                                glusterd_free_snap_op (missed_snap_op);
+                                goto out;
+                        }
+                        match = _gf_true;
+                        break;
+                } else if ((snap_opinfo->brick_num ==
+                            missed_snap_op->brick_num) &&
+                            (snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE) &&
+                            (missed_snap_op->op ==
+                             GF_SNAP_OPTION_TYPE_DELETE)) {
+                        /* Optimizing create and delete entries for the same
+                         * brick and same node
+                         */
+                        gf_log (this->name, GF_LOG_INFO,
+                                "Updating missed snap status "
+                                "for %s:%d:%s:%d as DONE",
+                                missed_snapinfo->node_snap_info,
+                                snap_opinfo->brick_num,
+                                snap_opinfo->brick_path,
+                                snap_opinfo->op);
+                        snap_opinfo->status = GD_MISSED_SNAP_DONE;
+                        ret = 0;
+                        glusterd_free_snap_op (missed_snap_op);
+                        goto out;
+                }
+        }
+
+        if (match == _gf_true) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Duplicate entry. Not updating");
+                glusterd_free_snap_op (missed_snap_op);
+        } else {
+                list_add_tail (&missed_snap_op->snap_ops_list,
+                               &missed_snapinfo->snap_ops);
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Add new missed snap entry to the missed_snaps list. */
+int32_t
+glusterd_store_missed_snaps_list (char *missed_info, int32_t brick_num,
+                                  char *brick_path, int32_t snap_op,
+                                  int32_t snap_status)
+{
+        int32_t                        ret                         = -1;
+        glusterd_missed_snap_info     *missed_snapinfo             = NULL;
+        glusterd_snap_op_t            *missed_snap_op              = NULL;
+        glusterd_conf_t               *priv                        = NULL;
+        gf_boolean_t                   match                       = _gf_false;
+        gf_boolean_t                   free_missed_snap_info       = _gf_false;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(missed_info);
+        GF_ASSERT(brick_path);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* Create the snap_op object consisting of the *
+         * snap id and the op */
+        ret = glusterd_missed_snap_op_new (&missed_snap_op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create new missed snap object.");
+                ret = -1;
+                goto out;
+        }
+
+        missed_snap_op->brick_path = gf_strdup(brick_path);
+        if (!missed_snap_op->brick_path) {
+                ret = -1;
+                goto out;
+        }
+        missed_snap_op->brick_num = brick_num;
+        missed_snap_op->op = snap_op;
+        missed_snap_op->status = snap_status;
+
+        /* Look for other entries for the same node and same snap */
+        list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
+                             missed_snaps) {
+                if (!strcmp (missed_snapinfo->node_snap_info,
+                             missed_info)) {
+                        /* Found missed snapshot info for *
+                         * the same node and same snap */
+                        match = _gf_true;
+                        break;
+                }
+        }
+
+        if (match == _gf_false) {
+                /* First snap op missed for the brick */
+                ret = glusterd_missed_snapinfo_new (&missed_snapinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to create missed snapinfo");
+                        goto out;
+                }
+                free_missed_snap_info = _gf_true;
+                missed_snapinfo->node_snap_info = gf_strdup(missed_info);
+                if (!missed_snapinfo->node_snap_info) {
+                        ret = -1;
+                        goto out;
+                }
+
+                list_add_tail (&missed_snap_op->snap_ops_list,
+                               &missed_snapinfo->snap_ops);
+                list_add_tail (&missed_snapinfo->missed_snaps,
+                               &priv->missed_snaps_list);
+
+                ret = 0;
+                goto out;
+        } else {
+                ret = glusterd_update_missed_snap_entry (missed_snapinfo,
+                                                         missed_snap_op);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to update existing missed snap entry.");
+                        goto out;
+                }
+        }
+
+out:
+        if (ret) {
+                glusterd_free_snap_op (missed_snap_op);
+
+                if (missed_snapinfo &&
+                    (free_missed_snap_info == _gf_true)) {
+                        if (missed_snapinfo->node_snap_info)
+                                GF_FREE (missed_snapinfo->node_snap_info);
+
+                        GF_FREE (missed_snapinfo);
+                }
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Add  missing snap entries to the in-memory conf->missed_snap_list */
+int32_t
+glusterd_add_missed_snaps_to_list (dict_t *dict, int32_t missed_snap_count)
+{
+        char                          *buf                         = NULL;
+        char                          *tmp                         = NULL;
+        char                          *save_ptr                    = NULL;
+        char                          *nodeid                      = NULL;
+        char                          *snap_uuid                   = NULL;
+        char                          *brick_path                  = NULL;
+        char                           missed_info[PATH_MAX]       = "";
+        char                           name_buf[PATH_MAX]          = "";
+        int32_t                        i                           = -1;
+        int32_t                        ret                         = -1;
+        int32_t                        brick_num                   = -1;
+        int32_t                        snap_op                     = -1;
+        int32_t                        snap_status                 = -1;
+        glusterd_conf_t               *priv                        = NULL;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* We can update the missed_snaps_list without acquiring *
+         * any additional locks as big lock will be held.        */
+        for (i = 0; i < missed_snap_count; i++) {
+                snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
+                          i);
+                ret = dict_get_str (dict, name_buf, &buf);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to fetch %s", name_buf);
+                        goto out;
+                }
+
+                gf_log (this->name, GF_LOG_DEBUG, "missed_snap_entry = %s",
+                        buf);
+
+                /* Need to make a duplicate string coz the same dictionary *
+                 * is resent to the non-originator nodes */
+                tmp = gf_strdup (buf);
+                if (!tmp) {
+                        ret = -1;
+                        goto out;
+                }
+
+                /* Fetch the node-id, snap-id, brick_num,
+                 * brick_path, snap_op and snap status
+                 */
+                nodeid = strtok_r (tmp, ":", &save_ptr);
+                snap_uuid = strtok_r (NULL, "=", &save_ptr);
+                brick_num = atoi(strtok_r (NULL, ":", &save_ptr));
+                brick_path = strtok_r (NULL, ":", &save_ptr);
+                snap_op = atoi(strtok_r (NULL, ":", &save_ptr));
+                snap_status = atoi(strtok_r (NULL, ":", &save_ptr));
+
+                if (!nodeid || !snap_uuid || !brick_path ||
+                    brick_num < 1 || snap_op < 1 ||
+                    snap_status < 1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Invalid missed_snap_entry");
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (missed_info, sizeof(missed_info), "%s:%s",
+                          nodeid, snap_uuid);
+
+                ret = glusterd_store_missed_snaps_list (missed_info,
+                                                        brick_num,
+                                                        brick_path,
+                                                        snap_op,
+                                                        snap_status);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to store missed snaps_list");
+                        goto out;
+                }
+
+                GF_FREE (tmp);
+                tmp = NULL;
+        }
+
+out:
+        if (tmp)
+                GF_FREE (tmp);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 2e6dd6b43..1c2ec58e8 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2007-2013 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -46,7 +46,7 @@
 #include <dirent.h>
 
 void
-glusterd_replace_slash_with_hipen (char *str)
+glusterd_replace_slash_with_hyphen (char *str)
 {
         char                    *ptr = NULL;
 
@@ -85,7 +85,7 @@ glusterd_store_key_vol_brick_set (glusterd_brickinfo_t *brickinfo,
         GF_ASSERT (len >= PATH_MAX);
 
         snprintf (key_vol_brick, len, "%s", brickinfo->path);
-        glusterd_replace_slash_with_hipen (key_vol_brick);
+        glusterd_replace_slash_with_hyphen (key_vol_brick);
 }
 
 static void
@@ -241,6 +241,25 @@ glusterd_store_brickinfo_write (int fd, glusterd_brickinfo_t *brickinfo)
         if (ret)
                 goto out;
 
+        if (strlen(brickinfo->device_path) > 0) {
+                snprintf (value, sizeof(value), "%s", brickinfo->device_path);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH,
+                                           value);
+                if (ret)
+                        goto out;
+        }
+
+        snprintf (value, sizeof(value), "%d", brickinfo->snap_status);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
+                                   value);
+        if (ret)
+                goto out;
+
+        if (!brickinfo->vg[0])
+                goto out;
+
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_VGNAME,
+                                         brickinfo->vg);
 out:
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
@@ -297,6 +316,7 @@ glusterd_store_brickinfo (glusterd_volinfo_t *volinfo,
                 goto out;
 
         ret = glusterd_store_perform_brick_store (brickinfo);
+
 out:
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
@@ -501,14 +521,13 @@ int _storeopts (dict_t *this, char *key, data_t *value, void *data)
 int32_t
 glusterd_volume_exclude_options_write (int fd, glusterd_volinfo_t *volinfo)
 {
-        char      *str   = NULL;
+        char    *str            = NULL;
+        char    buf[PATH_MAX]   = {0,};
+        int32_t ret             = -1;
 
         GF_ASSERT (fd > 0);
         GF_ASSERT (volinfo);
 
-        char                    buf[PATH_MAX] = {0,};
-        int32_t                 ret           = -1;
-
         snprintf (buf, sizeof (buf), "%d", volinfo->type);
         ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TYPE, buf);
         if (ret)
@@ -550,6 +569,14 @@ glusterd_volume_exclude_options_write (int fd, glusterd_volinfo_t *volinfo)
         if (ret)
                 goto out;
 
+        snprintf (buf, sizeof (buf), "%s", volinfo->parent_volname);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PARENT_VOLNAME, buf);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to store "
+                        GLUSTERD_STORE_KEY_PARENT_VOLNAME);
+                goto out;
+        }
+
         ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_ID,
                                    uuid_utoa (volinfo->volume_id));
         if (ret)
@@ -571,14 +598,41 @@ glusterd_volume_exclude_options_write (int fd, glusterd_volinfo_t *volinfo)
                         goto out;
         }
 
-        if (volinfo->backend == GD_VOL_BK_BD)  {
-                snprintf (buf, sizeof (buf), "%d", volinfo->backend);
-                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_BACKEND,
+        snprintf (buf, sizeof (buf), "%d", volinfo->op_version);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_OP_VERSION, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->client_op_version);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
+                                   buf);
+        if (ret)
+                goto out;
+        if (volinfo->caps) {
+                snprintf (buf, sizeof (buf), "%d", volinfo->caps);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_CAPS,
                                            buf);
                 if (ret)
                         goto out;
         }
 
+        snprintf (buf, sizeof (buf), "%d", volinfo->is_volume_restored);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_IS_RESTORED, buf);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Unable to write is_volume_restored");
+                goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%"PRIu64, volinfo->snap_max_hard_limit);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                                   buf);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Unable to write snap-max-hard-limit");
+                goto out;
+        }
+
 out:
         if (ret)
                 gf_log (THIS->name, GF_LOG_ERROR, "Unable to write volume "
@@ -596,8 +650,7 @@ glusterd_store_voldirpath_set (glusterd_volinfo_t *volinfo, char *voldirpath,
         priv = THIS->private;
         GF_ASSERT (priv);
 
-        snprintf (voldirpath, len, "%s/%s/%s", priv->workdir,
-                  GLUSTERD_VOLUME_DIR_PREFIX, volinfo->volname);
+        GLUSTERD_GET_VOLUME_DIR (voldirpath, volinfo, priv);
 }
 
 static int32_t
@@ -611,10 +664,32 @@ glusterd_store_create_volume_dir (glusterd_volinfo_t *volinfo)
         glusterd_store_voldirpath_set (volinfo, voldirpath,
                                        sizeof (voldirpath));
         ret = gf_store_mkdir (voldirpath);
+
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
         return ret;
 }
 
+static int32_t
+glusterd_store_create_snap_dir (glusterd_snap_t *snap)
+{
+        int32_t            ret                   = -1;
+        char               snapdirpath[PATH_MAX] = {0,};
+        glusterd_conf_t   *priv                  = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snap);
+
+        GLUSTERD_GET_SNAP_DIR (snapdirpath, snap, priv);
+
+        ret = mkdir_p (snapdirpath, 0755, _gf_true);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create snaps dir "
+                        "%s", snapdirpath);
+        }
+        return ret;
+}
+
 int32_t
 glusterd_store_volinfo_write (int fd, glusterd_volinfo_t *volinfo)
 {
@@ -639,6 +714,49 @@ out:
         return ret;
 }
 
+int32_t
+glusterd_store_snapinfo_write (glusterd_snap_t *snap)
+{
+        int32_t  ret            = -1;
+        int      fd             = 0;
+        char     buf[PATH_MAX]  = "";
+
+        GF_ASSERT (snap);
+
+        fd = gf_store_mkstemp (snap->shandle);
+        if (fd <= 0)
+               goto out;
+
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_ID,
+                                   uuid_utoa (snap->snap_id));
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", snap->snap_status);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_STATUS, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", snap->snap_restored);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_RESTORED, buf);
+        if (ret)
+                goto out;
+
+        if (snap->description) {
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_DESC,
+                                           snap->description);
+                if (ret)
+                        goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%ld", snap->time_stamp);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_TIMESTAMP, buf);
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
 static void
 glusterd_store_rbstatepath_set (glusterd_volinfo_t *volinfo, char *rbstatepath,
                                 size_t len)
@@ -683,6 +801,37 @@ glusterd_store_node_state_path_set (glusterd_volinfo_t *volinfo,
                   GLUSTERD_NODE_STATE_FILE);
 }
 
+static void
+glusterd_store_missed_snaps_list_path_set (char *missed_snaps_list,
+                                           size_t len)
+{
+        glusterd_conf_t *priv = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (missed_snaps_list);
+        GF_ASSERT (len <= PATH_MAX);
+
+        snprintf (missed_snaps_list, len, "%s/snaps/"
+                  GLUSTERD_MISSED_SNAPS_LIST_FILE, priv->workdir);
+}
+
+static void
+glusterd_store_snapfpath_set (glusterd_snap_t *snap, char *snap_fpath,
+                              size_t len)
+{
+        glusterd_conf_t *priv = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snap);
+        GF_ASSERT (snap_fpath);
+        GF_ASSERT (len <= PATH_MAX);
+
+        snprintf (snap_fpath, len, "%s/snaps/%s/%s", priv->workdir,
+                  snap->snapname, GLUSTERD_SNAP_INFO_FILE);
+}
+
 int32_t
 glusterd_store_create_rbstate_shandle_on_absence (glusterd_volinfo_t *volinfo)
 {
@@ -727,6 +876,42 @@ glusterd_store_create_nodestate_sh_on_absence (glusterd_volinfo_t *volinfo)
         return ret;
 }
 
+static int32_t
+glusterd_store_create_missed_snaps_list_shandle_on_absence ()
+{
+        char               missed_snaps_list[PATH_MAX] = "";
+        int32_t            ret                         = -1;
+        glusterd_conf_t   *priv                        = NULL;
+        xlator_t          *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        glusterd_store_missed_snaps_list_path_set (missed_snaps_list,
+                                                   sizeof(missed_snaps_list));
+
+        ret = gf_store_handle_create_on_absence
+                                            (&priv->missed_snaps_list_shandle,
+                                             missed_snaps_list);
+        return ret;
+}
+
+int32_t
+glusterd_store_create_snap_shandle_on_absence (glusterd_snap_t *snap)
+{
+        char            snapfpath[PATH_MAX] = {0};
+        int32_t         ret = 0;
+
+        GF_ASSERT (snap);
+
+        glusterd_store_snapfpath_set (snap, snapfpath, sizeof (snapfpath));
+        ret = gf_store_handle_create_on_absence (&snap->shandle, snapfpath);
+        return ret;
+}
+
 int32_t
 glusterd_store_brickinfos (glusterd_volinfo_t *volinfo, int vol_fd)
 {
@@ -974,6 +1159,14 @@ glusterd_store_volume_cleanup_tmp (glusterd_volinfo_t *volinfo)
         gf_store_unlink_tmppath (volinfo->node_state_shandle);
 }
 
+void
+glusterd_store_snap_cleanup_tmp (glusterd_snap_t *snap)
+{
+        GF_ASSERT (snap);
+
+        gf_store_unlink_tmppath (snap->shandle);
+}
+
 int32_t
 glusterd_store_brickinfos_atomic_update (glusterd_volinfo_t *volinfo)
 {
@@ -1025,6 +1218,60 @@ out:
 }
 
 int32_t
+glusterd_store_snap_atomic_update (glusterd_snap_t *snap)
+{
+        int ret = -1;
+        GF_ASSERT (snap);
+
+        ret = gf_store_rename_tmppath (snap->shandle);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't rename "
+                        "temporary file(s): Reason %s", strerror (errno));
+
+        return ret;
+}
+
+int32_t
+glusterd_store_snap (glusterd_snap_t *snap)
+{
+        int32_t                 ret = -1;
+
+        GF_ASSERT (snap);
+
+        ret = glusterd_store_create_snap_dir (snap);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create snap dir");
+                goto out;
+        }
+
+        ret = glusterd_store_create_snap_shandle_on_absence (snap);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create snap info "
+                        "file");
+                goto out;
+        }
+
+        ret = glusterd_store_snapinfo_write (snap);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to write snap info");
+                goto out;
+        }
+
+        ret = glusterd_store_snap_atomic_update (snap);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,"Failed to do automic update");
+                goto out;
+        }
+
+out:
+        if (ret)
+                glusterd_store_snap_cleanup_tmp (snap);
+
+        gf_log (THIS->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
 glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t ac)
 {
         int32_t                 ret = -1;
@@ -1081,7 +1328,6 @@ out:
         return ret;
 }
 
-
 int32_t
 glusterd_store_delete_volume (glusterd_volinfo_t *volinfo)
 {
@@ -1108,8 +1354,8 @@ glusterd_store_delete_volume (glusterd_volinfo_t *volinfo)
         GLUSTERD_GET_VOLUME_DIR (pathname, volinfo, priv);
 
         snprintf (delete_path, sizeof (delete_path),
-                  "%s/"GLUSTERD_TRASH"/%s.deleted", priv->workdir,
-                  uuid_utoa (volinfo->volume_id));
+        "%s/"GLUSTERD_TRASH"/%s.deleted", priv->workdir,
+        uuid_utoa (volinfo->volume_id));
 
         snprintf (trashdir, sizeof (trashdir), "%s/"GLUSTERD_TRASH,
                   priv->workdir);
@@ -1201,6 +1447,116 @@ out:
         return ret;
 }
 
+/*TODO: cleanup the duplicate code and implement a generic function for
+ * deleting snap/volume depending on the parameter flag */
+int32_t
+glusterd_store_delete_snap (glusterd_snap_t *snap)
+{
+        char             pathname[PATH_MAX]    = {0,};
+        int32_t          ret                   = 0;
+        glusterd_conf_t *priv                  = NULL;
+        DIR             *dir                   = NULL;
+        struct dirent   *entry                 = NULL;
+        char             path[PATH_MAX]        = {0,};
+        char             delete_path[PATH_MAX] = {0,};
+        char             trashdir[PATH_MAX]    = {0,};
+        struct stat      st                    = {0, };
+        xlator_t        *this                  = NULL;
+        gf_boolean_t     rename_fail           = _gf_false;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (snap);
+        GLUSTERD_GET_SNAP_DIR (pathname, snap, priv);
+
+        snprintf (delete_path, sizeof (delete_path),
+        "%s/"GLUSTERD_TRASH"/snap-%s.deleted", priv->workdir,
+        uuid_utoa (snap->snap_id));
+
+        snprintf (trashdir, sizeof (trashdir), "%s/"GLUSTERD_TRASH,
+                  priv->workdir);
+
+        ret = mkdir (trashdir, 0777);
+        if (ret && errno != EEXIST) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create trash "
+                        "directory, reason : %s", strerror (errno));
+                ret = -1;
+                goto out;
+        }
+
+        ret = rename (pathname, delete_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to rename snap "
+                        "directory %s to %s", snap->snapname, delete_path);
+                rename_fail = _gf_true;
+                goto out;
+        }
+
+        dir = opendir (delete_path);
+        if (!dir) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to open directory %s."
+                        " Reason : %s", delete_path, strerror (errno));
+                ret = 0;
+                goto out;
+        }
+
+        glusterd_for_each_entry (entry, dir);
+        while (entry) {
+                snprintf (path, PATH_MAX, "%s/%s", delete_path, entry->d_name);
+                ret = stat (path, &st);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG, "Failed to stat "
+                                "entry %s : %s", path, strerror (errno));
+                        goto stat_failed;
+                }
+
+                if (S_ISDIR (st.st_mode))
+                        ret = rmdir (path);
+                else
+                        ret = unlink (path);
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG, " Failed to remove "
+                                "%s. Reason : %s", path, strerror (errno));
+                }
+
+                gf_log (this->name, GF_LOG_DEBUG, "%s %s",
+                                ret ? "Failed to remove":"Removed",
+                                entry->d_name);
+stat_failed:
+                memset (path, 0, sizeof(path));
+                glusterd_for_each_entry (entry, dir);
+        }
+
+        ret = closedir (dir);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to close dir %s. "
+                        "Reason : %s",delete_path, strerror (errno));
+        }
+
+        ret = rmdir (delete_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to rmdir: %s,err: %s",
+                        delete_path, strerror (errno));
+        }
+        ret = rmdir (trashdir);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to rmdir: %s, Reason:"
+                        " %s", trashdir, strerror (errno));
+        }
+
+out:
+        if (snap->shandle) {
+                gf_store_handle_destroy (snap->shandle);
+                snap->shandle = NULL;
+        }
+        ret = (rename_fail == _gf_true) ? -1: 0;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
 
 int
 glusterd_store_global_info (xlator_t *this)
@@ -1211,6 +1567,7 @@ glusterd_store_global_info (xlator_t *this)
         char                    path[PATH_MAX]          = {0,};
         gf_store_handle_t       *handle                 = NULL;
         char                    *uuid_str               = NULL;
+        char                     buf[256]               = {0, };
 
         conf = this->private;
 
@@ -1263,6 +1620,24 @@ glusterd_store_global_info (xlator_t *this)
                 goto out;
         }
 
+        snprintf (buf, sizeof (buf), "%"PRIu64, conf->snap_max_hard_limit);
+        ret = gf_store_save_value (handle->fd,
+                                   GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT, buf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Storing snap-max-hard-limit failed ret = %d", ret);
+                goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%"PRIu64, conf->snap_max_soft_limit);
+        ret = gf_store_save_value (handle->fd,
+                                   GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT, buf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Storing snap-max-soft-limit failed ret = %d", ret);
+                goto out;
+        }
+
         ret = gf_store_rename_tmppath (handle);
 out:
         if (ret && (handle->fd > 0))
@@ -1334,15 +1709,94 @@ out:
         return ret;
 }
 
+int
+glusterd_retrieve_sys_snap_max_limit (xlator_t *this, uint64_t *limit,
+                                      char *key)
+{
+        char                    *limit_str      = NULL;
+        glusterd_conf_t         *priv           = NULL;
+        int                     ret             = -1;
+        uint64_t                tmp_limit       = 0;
+        char                    *tmp            = NULL;
+        char                    path[PATH_MAX]  = {0,};
+        gf_store_handle_t       *handle         = NULL;
+
+        GF_ASSERT (this);
+        priv = this->private;
+
+        GF_ASSERT (priv);
+        GF_ASSERT (limit);
+        GF_ASSERT (key);
+
+        if (!priv->handle) {
+                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
+                          GLUSTERD_INFO_FILE);
+                ret = gf_store_handle_retrieve (path, &handle);
+
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG, "Unable to get store "
+                                "handle!");
+                        goto out;
+                }
+
+                priv->handle = handle;
+        }
+
+        ret = gf_store_retrieve_value (priv->handle,
+                                       key,
+                                       &limit_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "No previous %s present", key);
+                goto out;
+        }
+
+        tmp_limit = strtoul (limit_str, &tmp, 10);
+        if ((tmp_limit <= 0) || (tmp && strlen (tmp) > 1)) {
+                gf_log (this->name, GF_LOG_WARNING, "invalid version number");
+                goto out;
+        }
+
+        *limit = tmp_limit;
+
+        ret = 0;
+out:
+        if (limit_str)
+                GF_FREE (limit_str);
+
+        return ret;
+}
 static int
 glusterd_restore_op_version (xlator_t *this)
 {
-        glusterd_conf_t *conf = NULL;
-        int ret = 0;
-        int op_version = 0;
+        glusterd_conf_t *conf           = NULL;
+        int              ret            = 0;
+        int              op_version     = 0;
 
         conf = this->private;
 
+        ret = glusterd_retrieve_sys_snap_max_limit (this,
+                                                    &conf->snap_max_hard_limit,
+                                                    GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Unable to retrieve system snap-max-hard-limit, "
+                        "setting it to default value(%d)",
+                        GLUSTERD_SNAPS_MAX_HARD_LIMIT);
+                conf->snap_max_hard_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+        }
+
+        ret = glusterd_retrieve_sys_snap_max_limit (this,
+                                                    &conf->snap_max_soft_limit,
+                                                    GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Unable to retrieve system snap-max-soft-limit, "
+                        "setting it to default value(%d)",
+                        GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT);
+                conf->snap_max_soft_limit = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
+        }
+
         ret = glusterd_retrieve_op_version (this, &op_version);
         if (!ret) {
                 if ((op_version < GD_OP_VERSION_MIN) ||
@@ -1431,7 +1885,6 @@ out:
 int32_t
 glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
 {
-
         int32_t                 ret = 0;
         glusterd_brickinfo_t    *brickinfo = NULL;
         gf_store_iter_t         *iter = NULL;
@@ -1452,10 +1905,9 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
 
         priv = THIS->private;
 
-        GLUSTERD_GET_BRICK_DIR (brickdir, volinfo, priv)
+        GLUSTERD_GET_BRICK_DIR (brickdir, volinfo, priv);
 
         ret = gf_store_iter_new (volinfo->shandle, &tmpiter);
-
         if (ret)
                 goto out;
 
@@ -1502,7 +1954,7 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                                     strlen (GLUSTERD_STORE_KEY_BRICK_PORT))) {
                                 gf_string2int (value, &brickinfo->port);
 
-                                if (brickinfo->port < GF_IANA_PRIV_PORTS_START){
+                                if (brickinfo->port < priv->base_port) {
                                         /* This is required to adhere to the
                                            IANA standards */
                                         brickinfo->port = 0;
@@ -1518,8 +1970,7 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                                     strlen (GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
                                 gf_string2int (value, &brickinfo->rdma_port);
 
-                                if (brickinfo->rdma_port <
-                                    GF_IANA_PRIV_PORTS_START){
+                                if (brickinfo->rdma_port < priv->base_port) {
                                         /* This is required to adhere to the
                                            IANA standards */
                                         brickinfo->rdma_port = 0;
@@ -1536,6 +1987,18 @@ glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
                         } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
                                              strlen (GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED))) {
                                 gf_string2int (value, &brickinfo->decommissioned);
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH,
+                                             strlen (GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH))) {
+                                strncpy (brickinfo->device_path, value,
+                                         sizeof (brickinfo->device_path));
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
+                                             strlen (GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS))) {
+                                gf_string2int (value, &brickinfo->snap_status);
+                        } else if (!strncmp (key,
+                                    GLUSTERD_STORE_KEY_BRICK_VGNAME,
+                                    strlen (GLUSTERD_STORE_KEY_BRICK_VGNAME))) {
+                                strncpy (brickinfo->vg, value,
+                                         sizeof (brickinfo->vg));
                         } else {
                                 gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
                                         key);
@@ -1572,10 +2035,9 @@ out:
 
 
 int32_t
-glusterd_store_retrieve_rbstate (char   *volname)
+glusterd_store_retrieve_rbstate (glusterd_volinfo_t *volinfo)
 {
         int32_t                   ret                   = -1;
-        glusterd_volinfo_t        *volinfo              = NULL;
         gf_store_iter_t           *iter                 = NULL;
         char                      *key                  = NULL;
         char                      *value                = NULL;
@@ -1583,15 +2045,13 @@ glusterd_store_retrieve_rbstate (char   *volname)
         glusterd_conf_t           *priv                 = NULL;
         char                      path[PATH_MAX]        = {0,};
         gf_store_op_errno_t       op_errno              = GD_STORE_SUCCESS;
+        xlator_t                 *this                  = NULL;
 
-        priv = THIS->private;
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't get"
-                        "volinfo for %s.", volname);
-                goto out;
-        }
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (volinfo);
 
         GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
         snprintf (path, sizeof (path), "%s/%s", volpath,
@@ -1668,16 +2128,15 @@ glusterd_store_retrieve_rbstate (char   *volname)
                 goto out;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
 
         return ret;
 }
 
 int32_t
-glusterd_store_retrieve_node_state (char   *volname)
+glusterd_store_retrieve_node_state (glusterd_volinfo_t *volinfo)
 {
         int32_t                   ret                   = -1;
-        glusterd_volinfo_t        *volinfo              = NULL;
         gf_store_iter_t           *iter                 = NULL;
         char                      *key                  = NULL;
         char                      *value                = NULL;
@@ -1685,15 +2144,13 @@ glusterd_store_retrieve_node_state (char   *volname)
         glusterd_conf_t           *priv                 = NULL;
         char                      path[PATH_MAX]        = {0,};
         gf_store_op_errno_t       op_errno              = GD_STORE_SUCCESS;
+        xlator_t                 *this                  = NULL;
 
-        priv = THIS->private;
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't get"
-                        "volinfo for %s.", volname);
-                goto out;
-        }
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (volinfo);
 
         GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
         snprintf (path, sizeof (path), "%s/%s", volpath,
@@ -1745,53 +2202,58 @@ glusterd_store_retrieve_node_state (char   *volname)
                 goto out;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
 
         return ret;
 }
 
-int32_t
-glusterd_store_retrieve_volume (char    *volname)
-{
-        int32_t                   ret                   = -1;
-        glusterd_volinfo_t        *volinfo              = NULL;
-        gf_store_iter_t           *iter                 = NULL;
-        char                      *key                  = NULL;
-        char                      *value                = NULL;
-        char                      volpath[PATH_MAX]     = {0,};
-        glusterd_conf_t           *priv                 = NULL;
-        char                      path[PATH_MAX]        = {0,};
-        int                       exists                = 0;
-        gf_store_op_errno_t       op_errno              = GD_STORE_SUCCESS;
-
-        ret = glusterd_volinfo_new (&volinfo);
 
-        if (ret)
-                goto out;
+int
+glusterd_store_update_volinfo (glusterd_volinfo_t *volinfo)
+{
+        int                     ret = -1;
+        int                     exists                  = 0;
+        char                    *key                    = NULL;
+        char                    *value                  = NULL;
+        char                    volpath[PATH_MAX]       = {0,};
+        char                    path[PATH_MAX]          = {0,};
+        xlator_t                *this                   = NULL;
+        glusterd_conf_t         *conf                   = NULL;
+        gf_store_iter_t         *iter                   = NULL;
+        gf_store_op_errno_t     op_errno                = GD_STORE_SUCCESS;
 
-        strncpy (volinfo->volname, volname, GLUSTERD_MAX_VOLUME_NAME);
+        this = THIS;
+        GF_ASSERT (this);
+        conf = THIS->private;
+        GF_ASSERT (volinfo);
 
-        priv = THIS->private;
+        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, conf);
 
-        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
         snprintf (path, sizeof (path), "%s/%s", volpath,
                   GLUSTERD_VOLUME_INFO_FILE);
 
         ret = gf_store_handle_retrieve (path, &volinfo->shandle);
-
-        if (ret)
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "volinfo handle is NULL");
                 goto out;
+        }
 
         ret = gf_store_iter_new (volinfo->shandle, &iter);
-
-        if (ret)
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get new store "
+                        "iter");
                 goto out;
+        }
 
         ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        if (ret)
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get next store "
+                        "iter");
                 goto out;
+        }
 
         while (!ret) {
+                gf_log ("", GF_LOG_DEBUG, "key = %s value = %s", key, value);
                 if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_TYPE,
                               strlen (GLUSTERD_STORE_KEY_VOL_TYPE))) {
                         volinfo->type = atoi (value);
@@ -1851,9 +2313,24 @@ glusterd_store_retrieve_volume (char    *volname)
                         gf_log ("", GF_LOG_DEBUG, "Parsed as "GEOREP" "
                                 " slave:key=%s,value:%s", key, value);
 
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_BACKEND,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_BACKEND))) {
-                        volinfo->backend = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_OP_VERSION,
+                                strlen (GLUSTERD_STORE_KEY_VOL_OP_VERSION))) {
+                        volinfo->op_version = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
+                                strlen (GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION))) {
+                        volinfo->client_op_version = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_CAPS,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_CAPS))) {
+                        volinfo->caps = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                                strlen (GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT))) {
+                        volinfo->snap_max_hard_limit = (uint64_t) atoll (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_IS_RESTORED,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_IS_RESTORED))) {
+                        volinfo->is_volume_restored = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_PARENT_VOLNAME,
+                                strlen (GLUSTERD_STORE_KEY_PARENT_VOLNAME))) {
+                        strncpy (volinfo->parent_volname, value, sizeof(volinfo->parent_volname) - 1);
                 } else {
 
                         if (is_key_glusterd_hooks_friendly (key)) {
@@ -1932,16 +2409,59 @@ glusterd_store_retrieve_volume (char    *volname)
                 volinfo->subvol_count = (volinfo->brick_count /
                                          volinfo->dist_leaf_count);
 
+                /* Only calculate volume op-versions if they are not found */
+                if (!volinfo->op_version && !volinfo->client_op_version)
+                        gd_update_volume_op_versions (volinfo);
         }
 
         if (op_errno != GD_STORE_EOF)
                 goto out;
 
         ret = gf_store_iter_destroy (iter);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to destroy store "
+                        "iter");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
 
+glusterd_volinfo_t*
+glusterd_store_retrieve_volume (char *volname, glusterd_snap_t *snap)
+{
+        int32_t                    ret                  = -1;
+        glusterd_volinfo_t        *volinfo              = NULL;
+        glusterd_volinfo_t        *origin_volinfo       = NULL;
+        glusterd_conf_t           *priv                 = NULL;
+        xlator_t                  *this                 = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (volname);
+
+        ret = glusterd_volinfo_new (&volinfo);
         if (ret)
                 goto out;
 
+        priv = THIS->private;
+
+        strncpy (volinfo->volname, volname, GLUSTERD_MAX_VOLUME_NAME);
+        volinfo->snapshot = snap;
+        if (snap)
+                volinfo->is_snap_volume = _gf_true;
+
+        ret = glusterd_store_update_volinfo (volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to update volinfo "
+                        "for %s volume", volname);
+                goto out;
+        }
+
         ret = glusterd_store_retrieve_bricks (volinfo);
         if (ret)
                 goto out;
@@ -1950,14 +2470,28 @@ glusterd_store_retrieve_volume (char    *volname)
         if (ret)
                 goto out;
 
-        gd_update_volume_op_versions (volinfo);
-
-        list_add_tail (&volinfo->vol_list, &priv->volumes);
-
+        if (!snap) {
+                list_add_tail (&volinfo->vol_list, &priv->volumes);
+        } else {
+                ret = glusterd_volinfo_find (volinfo->parent_volname,
+                                             &origin_volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Parent volinfo "
+                                "not found for %s volume", volname);
+                        goto out;
+                }
+                glusterd_list_add_snapvol (origin_volinfo, volinfo);
+        }
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        if (ret) {
+                if (volinfo)
+                        glusterd_volinfo_delete (volinfo);
+                volinfo = NULL;
+        }
 
-        return ret;
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+
+        return volinfo;
 }
 
 inline void
@@ -2057,9 +2591,9 @@ out:
 }
 
 int32_t
-glusterd_store_retrieve_volumes (xlator_t  *this)
+glusterd_store_retrieve_volumes (xlator_t  *this, glusterd_snap_t *snap)
 {
-        int32_t                ret              = 0;
+        int32_t                ret              = -1;
         char                   path[PATH_MAX]   = {0,};
         glusterd_conf_t       *priv             = NULL;
         DIR                   *dir              = NULL;
@@ -2071,51 +2605,414 @@ glusterd_store_retrieve_volumes (xlator_t  *this)
 
         GF_ASSERT (priv);
 
-        snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
-                        GLUSTERD_VOLUME_DIR_PREFIX);
+        if (snap)
+                snprintf (path, PATH_MAX, "%s/snaps/%s", priv->workdir,
+                          snap->snapname);
+        else
+                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
+                          GLUSTERD_VOLUME_DIR_PREFIX);
 
         dir = opendir (path);
 
         if (!dir) {
                 gf_log ("", GF_LOG_ERROR, "Unable to open dir %s", path);
-                ret = -1;
                 goto out;
         }
 
         glusterd_for_each_entry (entry, dir);
 
         while (entry) {
-                ret = glusterd_store_retrieve_volume (entry->d_name);
-                if (ret) {
+                if ( entry->d_type != DT_DIR )
+                        goto next;
+
+                volinfo = glusterd_store_retrieve_volume (entry->d_name, snap);
+                if (!volinfo) {
                         gf_log ("", GF_LOG_ERROR, "Unable to restore "
                                 "volume: %s", entry->d_name);
+                        ret = -1;
                         goto out;
                 }
 
-                ret = glusterd_store_retrieve_rbstate (entry->d_name);
+                ret = glusterd_store_retrieve_rbstate (volinfo);
                 if (ret) {
                         /* Backward compatibility */
                         gf_log ("", GF_LOG_INFO, "Creating a new rbstate "
                                 "for volume: %s.", entry->d_name);
-                        ret = glusterd_volinfo_find (entry->d_name, &volinfo);
                         ret = glusterd_store_create_rbstate_shandle_on_absence (volinfo);
                         ret = glusterd_store_perform_rbstate_store (volinfo);
                 }
 
-                ret = glusterd_store_retrieve_node_state (entry->d_name);
+                ret = glusterd_store_retrieve_node_state (volinfo);
                 if (ret) {
                         /* Backward compatibility */
                         gf_log ("", GF_LOG_INFO, "Creating a new node_state "
                                 "for volume: %s.", entry->d_name);
-                        ret = glusterd_volinfo_find (entry->d_name, &volinfo);
-                        ret =
                         glusterd_store_create_nodestate_sh_on_absence (volinfo);
                         ret = glusterd_store_perform_node_state_store (volinfo);
 
                 }
+
+next:
+                glusterd_for_each_entry (entry, dir);
+        }
+
+        ret = 0;
+out:
+        if (dir)
+                closedir (dir);
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+int32_t
+glusterd_resolve_snap_bricks (xlator_t  *this, glusterd_snap_t *snap)
+{
+        int32_t                 ret = -1;
+        glusterd_volinfo_t      *volinfo = NULL;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+
+        GF_ASSERT (this);
+        GF_VALIDATE_OR_GOTO (this->name, snap, out);
+
+        list_for_each_entry (volinfo, &snap->volumes, vol_list) {
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        ret = glusterd_resolve_brick (brickinfo);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "resolve brick failed in restore");
+                                goto out;
+                        }
+                }
+        }
+
+        ret = 0;
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+
+        return ret;
+}
+
+int
+glusterd_store_update_snap (glusterd_snap_t *snap)
+{
+        int                     ret = -1;
+        char                    *key                    = NULL;
+        char                    *value                  = NULL;
+        char                    snappath[PATH_MAX]      = {0,};
+        char                    path[PATH_MAX]          = {0,};
+        xlator_t                *this                   = NULL;
+        glusterd_conf_t         *conf                   = NULL;
+        gf_store_iter_t         *iter                   = NULL;
+        gf_store_op_errno_t     op_errno                = GD_STORE_SUCCESS;
+
+        this = THIS;
+        conf = this->private;
+        GF_ASSERT (snap);
+
+        GLUSTERD_GET_SNAP_DIR (snappath, snap, conf);
+
+        snprintf (path, sizeof (path), "%s/%s", snappath,
+                  GLUSTERD_SNAP_INFO_FILE);
+
+        ret = gf_store_handle_retrieve (path, &snap->shandle);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "snap handle is NULL");
+                goto out;
+        }
+
+        ret = gf_store_iter_new (snap->shandle, &iter);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get new store "
+                        "iter");
+                goto out;
+        }
+
+        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get next store "
+                        "iter");
+                goto out;
+        }
+
+        while (!ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "key = %s value = %s",
+                        key, value);
+
+                if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_ID,
+                                     strlen (GLUSTERD_STORE_KEY_SNAP_ID))) {
+                        ret = uuid_parse (value, snap->snap_id);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Failed to parse uuid");
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_RESTORED,
+                                   strlen (GLUSTERD_STORE_KEY_SNAP_RESTORED))) {
+                        snap->snap_restored = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_STATUS,
+                                     strlen (GLUSTERD_STORE_KEY_SNAP_STATUS))) {
+                        snap->snap_status = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_DESC,
+                                     strlen (GLUSTERD_STORE_KEY_SNAP_DESC))) {
+                        snap->description = gf_strdup (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_TIMESTAMP,
+                                  strlen (GLUSTERD_STORE_KEY_SNAP_TIMESTAMP))) {
+                        snap->time_stamp = atoi (value);
+                }
+
+                GF_FREE (key);
+                GF_FREE (value);
+                key = NULL;
+                value = NULL;
+
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        }
+
+        if (op_errno != GD_STORE_EOF)
+                goto out;
+
+        ret = gf_store_iter_destroy (iter);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to destroy store "
+                        "iter");
+        }
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_store_retrieve_snap (char *snapname)
+{
+        int32_t                ret    = -1;
+        dict_t                *dict   = NULL;
+        glusterd_snap_t       *snap   = NULL;
+        glusterd_conf_t       *priv   = NULL;
+        xlator_t              *this   = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snapname);
+
+        dict = dict_new();
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create dict");
+                ret = -1;
+                goto out;
+        }
+
+        snap = glusterd_new_snap_object ();
+        if (!snap) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create "
+                                                  " snap object");
+                goto out;
+        }
+
+        strncpy (snap->snapname, snapname, strlen(snapname));
+        ret = glusterd_store_update_snap (snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to update snapshot "
+                        "for %s snap", snapname);
+                goto out;
+        }
+
+        ret = glusterd_store_retrieve_volumes (this, snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to retrieve "
+                        "snap volumes for snap %s", snapname);
+                goto out;
+        }
+
+        /* Unlike bricks of normal volumes which are resolved at the end of
+           the glusterd restore, the bricks belonging to the snap volumes of
+           each snap should be resolved as part of snapshot restore itself.
+           Because if the snapshot has to be removed, then resolving bricks
+           helps glusterd in understanding what all bricks have its own uuid
+           and killing those bricks.
+        */
+        ret = glusterd_resolve_snap_bricks (this, snap);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING, "resolving the snap bricks"
+                        " failed (snap: %s)", snap?snap->snapname:"");
+
+        /* When the snapshot command from cli is received, the on disk and
+           in memory structures for the snapshot are created (with the status)
+           being marked as GD_SNAP_STATUS_INIT. Once the backend snapshot is
+           taken, the status is changed to GD_SNAP_STATUS_IN_USE. If glusterd
+           dies after taking the backend snapshot, but before updating the
+           status, then when glusterd comes up, it should treat that snapshot
+           as a failed snapshot and clean it up.
+        */
+        if (snap->snap_status != GD_SNAP_STATUS_IN_USE) {
+                ret = glusterd_snap_remove (dict, snap, _gf_true, _gf_true);
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING, "failed to remove"
+                                " the snapshot %s", snap->snapname);
+                goto out;
+        }
+
+        /* TODO: list_add_order can do 'N-square' comparisions and
+           is not efficient. Find a better solution to store the snap
+           in order */
+        list_add_order (&snap->snap_list, &priv->snapshots,
+                        glusterd_compare_snap_time);
+
+out:
+        if (dict)
+                dict_unref (dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+        return ret;
+}
+
+/* Read the missed_snap_list and update the in-memory structs */
+int32_t
+glusterd_store_retrieve_missed_snaps_list (xlator_t  *this)
+{
+        char                   buf[PATH_MAX]    = "";
+        char                   path[PATH_MAX]   = "";
+        char                  *missed_node_info = NULL;
+        char                  *brick_path       = NULL;
+        char                  *value            = NULL;
+        char                  *save_ptr         = NULL;
+        FILE                  *fp               = NULL;
+        int32_t                brick_num        = -1;
+        int32_t                snap_op          = -1;
+        int32_t                snap_status      = -1;
+        int32_t                ret              = -1;
+        glusterd_conf_t       *priv             = NULL;
+        gf_store_op_errno_t    store_errno      = GD_STORE_SUCCESS;
+
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* Get the path of the missed_snap_list */
+        glusterd_store_missed_snaps_list_path_set (path, sizeof(path));
+
+        fp = fopen (path, "r");
+        if (!fp) {
+                /* If errno is ENOENT then there are no missed snaps yet */
+                if (errno != ENOENT) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to open %s. "
+                                "Error: %s", path, strerror(errno));
+                } else {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "No missed snaps list.");
+                        ret = 0;
+                }
+                goto out;
+        }
+
+        do {
+                ret = gf_store_read_and_tokenize (fp, buf,
+                                                  &missed_node_info, &value,
+                                                  &store_errno);
+                if (ret) {
+                        if (store_errno == GD_STORE_EOF) {
+                                gf_log (this->name,
+                                        GF_LOG_DEBUG,
+                                        "EOF for missed_snap_list");
+                                ret = 0;
+                                break;
+                        }
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to fetch data from "
+                                "missed_snaps_list. Error: %s",
+                                gf_store_strerror (store_errno));
+                        goto out;
+                }
+
+                /* Fetch the brick_num, brick_path, snap_op and snap status */
+                brick_num = atoi(strtok_r (value, ":", &save_ptr));
+                brick_path = strtok_r (NULL, ":", &save_ptr);
+                snap_op = atoi(strtok_r (NULL, ":", &save_ptr));
+                snap_status = atoi(strtok_r (NULL, ":", &save_ptr));
+
+                if (!missed_node_info || !brick_path ||
+                    brick_num < 1 || snap_op < 1 ||
+                    snap_status < 1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Invalid missed_snap_entry");
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_store_missed_snaps_list (missed_node_info,
+                                                        brick_num,
+                                                        brick_path,
+                                                        snap_op,
+                                                        snap_status);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to store missed snaps_list");
+                        goto out;
+                }
+
+        } while (store_errno == GD_STORE_SUCCESS);
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_retrieve_snaps (xlator_t  *this)
+{
+        int32_t                ret              = 0;
+        char                   path[PATH_MAX]   = {0,};
+        glusterd_conf_t       *priv             = NULL;
+        DIR                   *dir              = NULL;
+        struct dirent         *entry            = NULL;
+
+        GF_ASSERT (this);
+        priv = this->private;
+
+        GF_ASSERT (priv);
+
+        snprintf (path, PATH_MAX, "%s/snaps", priv->workdir);
+
+        dir = opendir (path);
+
+        if (!dir) {
+                /* If snaps dir doesn't exists ignore the error for
+                   backward compatibility */
+                if (errno != ENOENT) {
+                        ret = -1;
+                        gf_log ("", GF_LOG_ERROR, "Unable to open dir %s", path);
+                }
+                goto out;
+        }
+
+        glusterd_for_each_entry (entry, dir);
+
+        while (entry) {
+                if (entry->d_type == DT_DIR) {
+                        ret = glusterd_store_retrieve_snap (entry->d_name);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to restore snapshot: %s",
+                                        entry->d_name);
+                                goto out;
+                        }
+                }
+
                 glusterd_for_each_entry (entry, dir);
         }
 
+        /* Retrieve missed_snaps_list */
+        ret = glusterd_store_retrieve_missed_snaps_list (this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Failed to retrieve missed_snaps_list");
+                goto out;
+        }
+
 out:
         if (dir)
                 closedir (dir);
@@ -2124,6 +3021,126 @@ out:
         return ret;
 }
 
+/* Writes all the contents of conf->missed_snap_list */
+int32_t
+glusterd_store_write_missed_snapinfo (int32_t fd)
+{
+        char                           value[PATH_MAX]             = "";
+        int32_t                        ret                         = -1;
+        glusterd_conf_t               *priv                        = NULL;
+        glusterd_missed_snap_info     *missed_snapinfo             = NULL;
+        glusterd_snap_op_t            *snap_opinfo                 = NULL;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* Write the missed_snap_entry */
+        list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
+                             missed_snaps) {
+                list_for_each_entry (snap_opinfo,
+                                     &missed_snapinfo->snap_ops,
+                                     snap_ops_list) {
+                        snprintf (value, sizeof(value), "%d:%s:%d:%d",
+                                  snap_opinfo->brick_num,
+                                  snap_opinfo->brick_path,
+                                  snap_opinfo->op, snap_opinfo->status);
+                        ret = gf_store_save_value
+                                           (fd,
+                                            missed_snapinfo->node_snap_info,
+                                            value);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to write missed snapinfo");
+                                goto out;
+                        }
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Adds the missed snap entries to the in-memory conf->missed_snap_list *
+ * and writes them to disk */
+int32_t
+glusterd_store_update_missed_snaps (dict_t *dict, int32_t missed_snap_count)
+{
+        int32_t                        fd                          = -1;
+        int32_t                        ret                         = -1;
+        glusterd_conf_t               *priv                        = NULL;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (missed_snap_count < 1) {
+                gf_log (this->name, GF_LOG_DEBUG, "No missed snaps");
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_store_create_missed_snaps_list_shandle_on_absence ();
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to obtain "
+                        "missed_snaps_list store handle.");
+                goto out;
+        }
+
+        fd = gf_store_mkstemp (priv->missed_snaps_list_shandle);
+        if (fd <= 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create tmp file");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_add_missed_snaps_to_list (dict, missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to add missed snaps to list");
+                goto out;
+        }
+
+        ret = glusterd_store_write_missed_snapinfo (fd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to write missed snaps to disk");
+                goto out;
+        }
+
+        ret = gf_store_rename_tmppath (priv->missed_snaps_list_shandle);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to rename the tmp file");
+                goto out;
+        }
+out:
+        if (ret && (fd > 0)) {
+                ret = gf_store_unlink_tmppath (priv->missed_snaps_list_shandle);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to unlink the tmp file");
+                }
+                ret = -1;
+        }
+
+        if (fd > 0)
+                close (fd);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
 int32_t
 glusterd_store_delete_peerinfo (glusterd_peerinfo_t *peerinfo)
 {
@@ -2525,7 +3542,11 @@ glusterd_restore ()
                 goto out;
         }
 
-        ret = glusterd_store_retrieve_volumes (this);
+        ret = glusterd_store_retrieve_volumes (this, NULL);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_retrieve_snaps (this);
         if (ret)
                 goto out;
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h
index e4f9e4a0a..1b5cebc0c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.h
+++ b/xlators/mgmt/glusterd/src/glusterd-store.h
@@ -35,38 +35,55 @@ typedef enum glusterd_store_ver_ac_{
 } glusterd_volinfo_ver_ac_t;
 
 
-#define GLUSTERD_STORE_UUID_KEY           "UUID"
-
-#define GLUSTERD_STORE_KEY_VOL_TYPE       "type"
-#define GLUSTERD_STORE_KEY_VOL_COUNT      "count"
-#define GLUSTERD_STORE_KEY_VOL_STATUS     "status"
-#define GLUSTERD_STORE_KEY_VOL_PORT       "port"
-#define GLUSTERD_STORE_KEY_VOL_SUB_COUNT  "sub_count"
-#define GLUSTERD_STORE_KEY_VOL_STRIPE_CNT  "stripe_count"
-#define GLUSTERD_STORE_KEY_VOL_REPLICA_CNT "replica_count"
-#define GLUSTERD_STORE_KEY_VOL_BRICK      "brick"
-#define GLUSTERD_STORE_KEY_VOL_VERSION    "version"
-#define GLUSTERD_STORE_KEY_VOL_TRANSPORT  "transport-type"
-#define GLUSTERD_STORE_KEY_VOL_ID         "volume-id"
-#define GLUSTERD_STORE_KEY_RB_STATUS      "rb_status"
-#define GLUSTERD_STORE_KEY_RB_SRC_BRICK   "rb_src"
-#define GLUSTERD_STORE_KEY_RB_DST_BRICK   "rb_dst"
-#define GLUSTERD_STORE_KEY_RB_DST_PORT    "rb_port"
-#define GLUSTERD_STORE_KEY_VOL_DEFRAG     "rebalance_status"
-#define GLUSTERD_STORE_KEY_DEFRAG_OP      "rebalance_op"
-#define GLUSTERD_STORE_KEY_USERNAME       "username"
-#define GLUSTERD_STORE_KEY_PASSWORD       "password"
-
-#define GLUSTERD_STORE_KEY_BRICK_HOSTNAME "hostname"
-#define GLUSTERD_STORE_KEY_BRICK_PATH     "path"
-#define GLUSTERD_STORE_KEY_BRICK_PORT     "listen-port"
-#define GLUSTERD_STORE_KEY_BRICK_RDMA_PORT "rdma.listen-port"
+#define GLUSTERD_STORE_UUID_KEY                 "UUID"
+
+#define GLUSTERD_STORE_KEY_VOL_TYPE             "type"
+#define GLUSTERD_STORE_KEY_VOL_COUNT            "count"
+#define GLUSTERD_STORE_KEY_VOL_STATUS           "status"
+#define GLUSTERD_STORE_KEY_VOL_PORT             "port"
+#define GLUSTERD_STORE_KEY_VOL_SUB_COUNT        "sub_count"
+#define GLUSTERD_STORE_KEY_VOL_STRIPE_CNT       "stripe_count"
+#define GLUSTERD_STORE_KEY_VOL_REPLICA_CNT      "replica_count"
+#define GLUSTERD_STORE_KEY_VOL_BRICK            "brick"
+#define GLUSTERD_STORE_KEY_VOL_VERSION          "version"
+#define GLUSTERD_STORE_KEY_VOL_TRANSPORT        "transport-type"
+#define GLUSTERD_STORE_KEY_VOL_ID               "volume-id"
+#define GLUSTERD_STORE_KEY_VOL_IS_RESTORED      "is-volume-restored"
+#define GLUSTERD_STORE_KEY_RB_STATUS            "rb_status"
+#define GLUSTERD_STORE_KEY_RB_SRC_BRICK         "rb_src"
+#define GLUSTERD_STORE_KEY_RB_DST_BRICK         "rb_dst"
+#define GLUSTERD_STORE_KEY_RB_DST_PORT          "rb_port"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG           "rebalance_status"
+#define GLUSTERD_STORE_KEY_DEFRAG_OP            "rebalance_op"
+#define GLUSTERD_STORE_KEY_USERNAME             "username"
+#define GLUSTERD_STORE_KEY_PASSWORD             "password"
+#define GLUSTERD_STORE_KEY_PARENT_VOLNAME       "parent_volname"
+#define GLUSTERD_STORE_KEY_VOL_OP_VERSION       "op-version"
+#define GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION "client-op-version"
+
+#define GLUSTERD_STORE_KEY_SNAP_NAME            "name"
+#define GLUSTERD_STORE_KEY_SNAP_ID              "snap-id"
+#define GLUSTERD_STORE_KEY_SNAP_DESC            "desc"
+#define GLUSTERD_STORE_KEY_SNAP_TIMESTAMP       "time-stamp"
+#define GLUSTERD_STORE_KEY_SNAP_STATUS          "status"
+#define GLUSTERD_STORE_KEY_SNAP_RESTORED        "snap-restored"
+#define GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT  "snap-max-hard-limit"
+#define GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT  "snap-max-soft-limit"
+
+#define GLUSTERD_STORE_KEY_BRICK_HOSTNAME       "hostname"
+#define GLUSTERD_STORE_KEY_BRICK_PATH           "path"
+#define GLUSTERD_STORE_KEY_BRICK_PORT           "listen-port"
+#define GLUSTERD_STORE_KEY_BRICK_RDMA_PORT      "rdma.listen-port"
 #define GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED "decommissioned"
+#define GLUSTERD_STORE_KEY_BRICK_VGNAME         "vg"
+#define GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH    "device_path"
+#define GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS    "snap-status"
 
-#define GLUSTERD_STORE_KEY_PEER_UUID      "uuid"
-#define GLUSTERD_STORE_KEY_PEER_HOSTNAME  "hostname"
-#define GLUSTERD_STORE_KEY_PEER_STATE     "state"
-#define GLUSTERD_STORE_KEY_VOL_BACKEND    "backend"
+#define GLUSTERD_STORE_KEY_PEER_UUID            "uuid"
+#define GLUSTERD_STORE_KEY_PEER_HOSTNAME        "hostname"
+#define GLUSTERD_STORE_KEY_PEER_STATE           "state"
+
+#define GLUSTERD_STORE_KEY_VOL_CAPS             "caps"
 
 #define glusterd_for_each_entry(entry, dir) \
         do {\
@@ -89,6 +106,9 @@ int32_t
 glusterd_store_delete_volume (glusterd_volinfo_t *volinfo);
 
 int32_t
+glusterd_store_delete_snap (glusterd_snap_t *snap);
+
+int32_t
 glusterd_retrieve_uuid ();
 
 int32_t
@@ -123,5 +143,22 @@ int32_t
 glusterd_store_retrieve_options (xlator_t *this);
 
 int32_t
+glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo);
+
+int32_t
 glusterd_store_options (xlator_t *this, dict_t *opts);
+
+void
+glusterd_replace_slash_with_hyphen (char *str);
+
+int32_t
+glusterd_store_perform_volume_store (glusterd_volinfo_t *volinfo);
+
+int32_t
+glusterd_store_snap (glusterd_snap_t *snap);
+
+int32_t
+glusterd_store_update_missed_snaps (dict_t *dict,
+                                    int32_t missed_snap_count);
+
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
index a694cae84..438df8266 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.c
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -17,8 +17,11 @@
 #include "glusterd.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
+#include "glusterd-locks.h"
 
-static inline void
+extern glusterd_op_info_t opinfo;
+
+void
 gd_synctask_barrier_wait (struct syncargs *args, int count)
 {
         glusterd_conf_t *conf = THIS->private;
@@ -31,9 +34,9 @@ gd_synctask_barrier_wait (struct syncargs *args, int count)
 }
 
 static void
-gd_collate_errors (struct syncargs *args, int op_ret, int op_errno,
-                   char *op_errstr, int op_code,
-                   glusterd_peerinfo_t *peerinfo, u_char *uuid)
+gd_mgmt_v3_collate_errors (struct syncargs *args, int op_ret, int op_errno,
+                           char *op_errstr, int op_code,
+                           glusterd_peerinfo_t *peerinfo, u_char *uuid)
 {
         char     err_str[PATH_MAX] = "Please check log file for details.";
         char     op_err[PATH_MAX] = "";
@@ -56,13 +59,67 @@ gd_collate_errors (struct syncargs *args, int op_ret, int op_errno,
                 }
 
                 switch (op_code){
-                        case GLUSTERD_MGMT_CLUSTER_LOCK :
+                        case GLUSTERD_MGMT_V3_LOCK:
                         {
                                 len = snprintf (op_err, sizeof(op_err) - 1,
-                                                "Locking failed on %s. %s",
-                                                peer_str, err_str);
+                                                "Locking failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                        case GLUSTERD_MGMT_V3_UNLOCK:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Unlocking failed "
+                                                "on %s. %s", peer_str, err_str);
                                 break;
                         }
+                }
+                op_err[len] = '\0';
+
+                if (args->errstr) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s\n%s", args->errstr,
+                                        op_err);
+                        GF_FREE (args->errstr);
+                        args->errstr = NULL;
+                } else
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s", op_err);
+                err_str[len] = '\0';
+
+                gf_log ("", GF_LOG_ERROR, "%s", op_err);
+                args->errstr = gf_strdup (err_str);
+        }
+
+        return;
+}
+
+static void
+gd_collate_errors (struct syncargs *args, int op_ret, int op_errno,
+                   char *op_errstr, int op_code,
+                   glusterd_peerinfo_t *peerinfo, u_char *uuid)
+{
+        char     err_str[PATH_MAX] = "Please check log file for details.";
+        char     op_err[PATH_MAX] = "";
+        int      len               = -1;
+        char    *peer_str      = NULL;
+
+        if (op_ret) {
+                args->op_ret = op_ret;
+                args->op_errno = op_errno;
+
+                if (peerinfo)
+                        peer_str = peerinfo->hostname;
+                else
+                        peer_str = uuid_utoa (uuid);
+
+                if (op_errstr && strcmp (op_errstr, "")) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "Error: %s", op_errstr);
+                        err_str[len] = '\0';
+                }
+
+                switch (op_code){
                         case GLUSTERD_MGMT_CLUSTER_UNLOCK :
                         {
                                 len = snprintf (op_err, sizeof(op_err) - 1,
@@ -105,7 +162,7 @@ gd_collate_errors (struct syncargs *args, int op_ret, int op_errno,
         return;
 }
 
-static void
+void
 gd_syncargs_init (struct syncargs *args, dict_t *op_ctx)
 {
         args->dict = op_ctx;
@@ -207,8 +264,9 @@ out:
 /* Defined in glusterd-rpc-ops.c */
 extern struct rpc_clnt_program gd_mgmt_prog;
 extern struct rpc_clnt_program gd_brick_prog;
+extern struct rpc_clnt_program gd_mgmt_v3_prog;
 
-static int
+int
 glusterd_syncop_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp)
 {
         int ret = 0;
@@ -275,6 +333,12 @@ glusterd_syncop_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp)
                         goto out;
         break;
 
+        case GD_OP_SNAP:
+                ret = glusterd_snap_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
         default:
         break;
         }
@@ -353,6 +417,183 @@ gd_syncop_mgmt_lock (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
 }
 
 int32_t
+gd_syncop_mgmt_v3_lock_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                               int count, void *myframe)
+{
+        int                         ret         = -1;
+        struct syncargs             *args       = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        gd1_mgmt_v3_lock_rsp         rsp        = {{0},};
+        call_frame_t                *frame      = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        GF_ASSERT(req);
+        GF_ASSERT(iov);
+        GF_ASSERT(myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_LOCK,
+                                   peerinfo, rsp.uuid);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_mgmt_v3_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                            int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_syncop_mgmt_v3_lock_cbk_fn);
+}
+
+int
+gd_syncop_mgmt_v3_lock (glusterd_op_t op, dict_t *op_ctx,
+                        glusterd_peerinfo_t *peerinfo,
+                        struct syncargs *args, uuid_t my_uuid,
+                        uuid_t recv_uuid, uuid_t txn_id)
+{
+        int                      ret  = -1;
+        gd1_mgmt_v3_lock_req     req  = {{0},};
+        glusterd_conf_t         *conf = THIS->private;
+
+        GF_ASSERT(op_ctx);
+        GF_ASSERT(peerinfo);
+        GF_ASSERT(args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        uuid_copy (req.txn_id, txn_id);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_LOCK,
+                                        gd_syncop_mgmt_v3_lock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_lock_req);
+        synclock_lock (&conf->big_lock);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+gd_syncop_mgmt_v3_unlock_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        int                         ret         = -1;
+        struct syncargs             *args       = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        gd1_mgmt_v3_unlock_rsp       rsp        = {{0},};
+        call_frame_t                *frame      = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        GF_ASSERT(req);
+        GF_ASSERT(iov);
+        GF_ASSERT(myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        /* Set peer as locked, so we unlock only the locked peers */
+        if (rsp.op_ret == 0)
+                peerinfo->locked = _gf_true;
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_UNLOCK,
+                                   peerinfo, rsp.uuid);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_mgmt_v3_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                              int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_syncop_mgmt_v3_unlock_cbk_fn);
+}
+
+int
+gd_syncop_mgmt_v3_unlock (dict_t *op_ctx, glusterd_peerinfo_t *peerinfo,
+                          struct syncargs *args, uuid_t my_uuid,
+                          uuid_t recv_uuid, uuid_t txn_id)
+{
+        int                          ret  = -1;
+        gd1_mgmt_v3_unlock_req       req  = {{0},};
+        glusterd_conf_t             *conf = THIS->private;
+
+        GF_ASSERT(op_ctx);
+        GF_ASSERT(peerinfo);
+        GF_ASSERT(args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        uuid_copy (req.txn_id, txn_id);
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_UNLOCK,
+                                        gd_syncop_mgmt_v3_unlock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_unlock_req);
+        synclock_lock (&conf->big_lock);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
 _gd_syncop_mgmt_unlock_cbk (struct rpc_req *req, struct iovec *iov,
                             int count, void *myframe)
 {
@@ -641,10 +882,12 @@ gd_syncop_mgmt_brick_op (struct rpc_clnt *rpc, glusterd_pending_node_t *pnode,
         GD_SYNCOP (rpc, (&args), NULL, gd_syncop_brick_op_cbk, req,
                    &gd_brick_prog, req->op, xdr_gd1_mgmt_brick_op_req);
 
-        if (args.errstr && errstr)
-                *errstr = args.errstr;
-        else
-                GF_FREE (args.errstr);
+        if (args.errstr) {
+                if ((strlen(args.errstr) > 0) && errstr)
+                        *errstr = args.errstr;
+                else
+                        GF_FREE (args.errstr);
+        }
 
         if (GD_OP_STATUS_VOLUME == op) {
                 ret = dict_set_int32 (args.dict, "index", pnode->index);
@@ -814,8 +1057,8 @@ gd_build_peers_list (struct list_head *peers, struct list_head *xact_peers,
 }
 
 int
-gd_lock_op_phase (struct list_head *peers, glusterd_op_t op, dict_t *op_ctx,
-                  char **op_errstr, int npeers)
+gd_lock_op_phase (glusterd_conf_t  *conf, glusterd_op_t op, dict_t *op_ctx,
+                  char **op_errstr, int npeers, uuid_t txn_id)
 {
         int                 ret         = -1;
         int                 peer_cnt    = 0;
@@ -823,6 +1066,9 @@ gd_lock_op_phase (struct list_head *peers, glusterd_op_t op, dict_t *op_ctx,
         xlator_t            *this       = NULL;
         glusterd_peerinfo_t *peerinfo   = NULL;
         struct syncargs     args        = {0};
+        struct list_head   *peers       = NULL;
+
+        peers = &conf->xaction_peers;
 
         if (!npeers) {
                 ret = 0;
@@ -833,22 +1079,38 @@ gd_lock_op_phase (struct list_head *peers, glusterd_op_t op, dict_t *op_ctx,
         synctask_barrier_init((&args));
         peer_cnt = 0;
         list_for_each_entry (peerinfo, peers, op_peers_list) {
-                /* Reset lock status */
-                peerinfo->locked = _gf_false;
-                gd_syncop_mgmt_lock (peerinfo, &args, MY_UUID, peer_uuid);
+                if (conf->op_version < 3) {
+                        /* Reset lock status */
+                        peerinfo->locked = _gf_false;
+                        gd_syncop_mgmt_lock (peerinfo, &args,
+                                             MY_UUID, peer_uuid);
+                } else
+                        gd_syncop_mgmt_v3_lock (op, op_ctx, peerinfo, &args,
+                                                MY_UUID, peer_uuid, txn_id);
                 peer_cnt++;
         }
         gd_synctask_barrier_wait((&args), peer_cnt);
-        ret = args.op_ret;
-        if (ret) {
-                gf_asprintf (op_errstr, "Another transaction could be "
-                             "in progress. Please try again after "
-                             "sometime.");
-                gf_log (this->name, GF_LOG_ERROR, "Failed to acquire lock");
-                goto out;
+
+        if (args.op_ret) {
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+                else {
+                        ret = gf_asprintf (op_errstr, "Another transaction could be "
+                                           "in progress. Please try again after "
+                                           "sometime.");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to acquire lock");
+
+                }
         }
 
-        ret = 0;
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent lock op req for 'Volume %s' "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
 out:
         return ret;
 }
@@ -1016,9 +1278,10 @@ out:
 }
 
 int
-gd_unlock_op_phase (struct list_head *peers, glusterd_op_t op, int op_ret,
+gd_unlock_op_phase (glusterd_conf_t  *conf, glusterd_op_t op, int op_ret,
                     rpcsvc_request_t *req, dict_t *op_ctx, char *op_errstr,
-                    int npeers)
+                    int npeers, char *volname, gf_boolean_t is_acquired,
+                    uuid_t txn_id)
 {
         glusterd_peerinfo_t *peerinfo   = NULL;
         glusterd_peerinfo_t *tmp        = NULL;
@@ -1027,23 +1290,34 @@ gd_unlock_op_phase (struct list_head *peers, glusterd_op_t op, int op_ret,
         int                 ret         = -1;
         xlator_t            *this       = NULL;
         struct syncargs     args        = {0};
+        struct list_head   *peers       = NULL;
+
+        peers = &conf->xaction_peers;
 
         if (!npeers) {
                 ret = 0;
                 goto out;
         }
 
+        /* If the lock has not been held during this
+         * transaction, do not send unlock requests */
+        if (!is_acquired)
+                goto out;
+
         this = THIS;
         synctask_barrier_init((&args));
         peer_cnt = 0;
         list_for_each_entry_safe (peerinfo, tmp, peers, op_peers_list) {
-                /* Only unlock peers that were locked */
-                if (peerinfo->locked) {
-                        gd_syncop_mgmt_unlock (peerinfo, &args, MY_UUID,
-                                               tmp_uuid);
-                        peer_cnt++;
-                }
-
+                if (conf->op_version < 3) {
+                        /* Only unlock peers that were locked */
+                        if (peerinfo->locked)
+                                gd_syncop_mgmt_unlock (peerinfo, &args,
+                                                       MY_UUID, tmp_uuid);
+                } else
+                        gd_syncop_mgmt_v3_unlock (op_ctx, peerinfo,
+                                                  &args, MY_UUID,
+                                                  tmp_uuid, txn_id);
+                peer_cnt++;
                 list_del_init (&peerinfo->op_peers_list);
         }
         gd_synctask_barrier_wait((&args), peer_cnt);
@@ -1056,7 +1330,20 @@ gd_unlock_op_phase (struct list_head *peers, glusterd_op_t op, int op_ret,
 out:
         glusterd_op_send_cli_response (op, op_ret, 0, req, op_ctx, op_errstr);
         glusterd_op_clear_op (op);
-        glusterd_unlock (MY_UUID);
+        if (is_acquired) {
+                /* Based on the op-version, we release *
+                 * the cluster or mgmt_v3 lock */
+                if (conf->op_version < 3)
+                        glusterd_unlock (MY_UUID);
+                else {
+                        ret = glusterd_mgmt_v3_unlock (volname, MY_UUID,
+                                                       "vol");
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to release lock for %s",
+                                        volname);
+                }
+        }
 
         return 0;
 }
@@ -1073,7 +1360,8 @@ gd_get_brick_count (struct list_head *bricks)
 }
 
 int
-gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict, char **op_errstr)
+gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                   char **op_errstr)
 {
         glusterd_pending_node_t *pending_node = NULL;
         struct list_head        selected = {0,};
@@ -1145,20 +1433,63 @@ out:
 void
 gd_sync_task_begin (dict_t *op_ctx, rpcsvc_request_t * req)
 {
-        int                         ret             = -1;
-        int                         npeers          = 0;
-        dict_t                      *req_dict       = NULL;
-        glusterd_conf_t             *conf           = NULL;
-        glusterd_op_t               op              = 0;
-        int32_t                     tmp_op          = 0;
-        char                        *op_errstr      = NULL;
-        xlator_t                    *this           = NULL;
+        int                         ret              = -1;
+        int                         npeers           = 0;
+        dict_t                      *req_dict        = NULL;
+        glusterd_conf_t             *conf            = NULL;
+        glusterd_op_t               op               = 0;
+        int32_t                     tmp_op           = 0;
+        char                        *op_errstr       = NULL;
+        char                        *tmp             = NULL;
+        char                        *volname         = NULL;
+        xlator_t                    *this            = NULL;
+        gf_boolean_t                is_acquired      = _gf_false;
+        uuid_t                      *txn_id          = NULL;
+        uuid_t                      *originator_uuid = NULL;
+        glusterd_op_info_t          txn_opinfo;
 
         this = THIS;
         GF_ASSERT (this);
         conf = this->private;
         GF_ASSERT (conf);
 
+        /* Generate a transaction-id for this operation and
+         * save it in the dict */
+        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!txn_id) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_generate (*txn_id);
+
+        ret = dict_set_bin (op_ctx, "transaction_id",
+                            txn_id, sizeof(*txn_id));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set transaction id.");
+                goto out;
+        } else
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Transaction_id = %s", uuid_utoa (*txn_id));
+
+        /* Save the MY_UUID as the originator_uuid */
+        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
+                                     gf_common_mt_uuid_t);
+        if (!originator_uuid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*originator_uuid, MY_UUID);
+        ret = dict_set_bin (op_ctx, "originator_uuid",
+                            originator_uuid, sizeof (uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set originator_uuid.");
+                goto out;
+        }
+
         ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, &tmp_op);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Failed to get volume "
@@ -1167,24 +1498,77 @@ gd_sync_task_begin (dict_t *op_ctx, rpcsvc_request_t * req)
         }
 
         op = tmp_op;
-        ret = glusterd_lock (MY_UUID);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to acquire lock");
-                gf_asprintf (&op_errstr, "Another transaction is in progress. "
-                             "Please try again after sometime.");
-                goto out;
+
+        /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
+        if (conf->op_version < 3) {
+                ret = glusterd_lock (MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock");
+                        gf_asprintf (&op_errstr,
+                                     "Another transaction is in progress. "
+                                     "Please try again after sometime.");
+                        goto out;
+                }
+        } else {
+
+                /* If no volname is given as a part of the command, locks will
+                 * not be held */
+                ret = dict_get_str (op_ctx, "volname", &tmp);
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG, "Failed to get volume "
+                                "name");
+                        goto local_locking_done;
+                } else {
+                        /* Use a copy of volname, as cli response will be
+                         * sent before the unlock, and the volname in the
+                         * dict, might be removed */
+                        volname = gf_strdup (tmp);
+                        if (!volname)
+                                goto out;
+                }
+
+                ret = glusterd_mgmt_v3_lock (volname, MY_UUID, "vol");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock for %s", volname);
+                        gf_asprintf (&op_errstr,
+                                     "Another transaction is in progress "
+                                     "for %s. Please try again after sometime.",
+                                     volname);
+                        goto out;
+                }
         }
 
-        /* storing op globally to access in synctask code paths
-         * This is still acceptable, as we are performing this under
-         * the 'cluster' lock*/
-        glusterd_op_set_op  (op);
-        INIT_LIST_HEAD (&conf->xaction_peers);
-        npeers = gd_build_peers_list  (&conf->peers, &conf->xaction_peers, op);
+        is_acquired = _gf_true;
+
+local_locking_done:
 
-        ret = gd_lock_op_phase (&conf->xaction_peers, op, op_ctx, &op_errstr, npeers);
+        /* Save opinfo for this transaction with the transaction id */
+        glusterd_txn_opinfo_init (&txn_opinfo, NULL, &op, NULL, NULL);
+        ret = glusterd_set_txn_opinfo (txn_id, &txn_opinfo);
         if (ret)
-                goto out;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+
+        opinfo = txn_opinfo;
+
+        INIT_LIST_HEAD (&conf->xaction_peers);
+
+        /* Make 'volume status tasks' command a local operation.
+         * This is accomplished by setting npeers to 0.
+         */
+        if (!glusterd_is_status_tasks_op (op, op_ctx))
+                npeers = gd_build_peers_list  (&conf->peers,
+                                               &conf->xaction_peers, op);
+
+        /* If no volname is given as a part of the command, locks will
+         * not be held */
+        if (volname) {
+                ret = gd_lock_op_phase (conf, op, op_ctx, &op_errstr, npeers, *txn_id);
+                if (ret)
+                        goto out;
+        }
 
         ret = glusterd_op_build_payload (&req_dict, &op_errstr, op_ctx);
         if (ret) {
@@ -1211,8 +1595,17 @@ gd_sync_task_begin (dict_t *op_ctx, rpcsvc_request_t * req)
 
         ret = 0;
 out:
-        (void) gd_unlock_op_phase (&conf->xaction_peers, op, ret, req,
-                                   op_ctx, op_errstr, npeers);
+        (void) gd_unlock_op_phase (conf, op, ret, req, op_ctx, op_errstr,
+                                   npeers, volname, is_acquired, *txn_id);
+
+        /* Clearing the transaction opinfo */
+        ret = glusterd_clear_txn_opinfo (txn_id);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to clear transaction's opinfo");
+
+        if (volname)
+                GF_FREE (volname);
 
         if (req_dict)
                 dict_unref (req_dict);
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.h b/xlators/mgmt/glusterd/src/glusterd-syncop.h
index 7e4d47f9a..e83ea2f4c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.h
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.h
@@ -12,6 +12,7 @@
 
 #include "syncop.h"
 #include "glusterd-sm.h"
+#include "glusterd.h"
 
 #define GD_SYNC_OPCODE_KEY "sync-mgmt-operation"
 
@@ -51,4 +52,20 @@ int gd_syncop_mgmt_stage_op (struct rpc_clnt *rpc, struct syncargs *arg,
 int gd_syncop_mgmt_commit_op (struct rpc_clnt *rpc, struct syncargs *arg,
                               uuid_t my_uuid, uuid_t recv_uuid, int op,
                               dict_t *dict_out, dict_t *op_ctx);
+
+void
+gd_synctask_barrier_wait (struct syncargs *args, int count);
+
+int
+gd_build_peers_list (struct list_head *peers, struct list_head *xact_peers,
+                     glusterd_op_t op);
+int
+gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                   char **op_errstr);
+
+int
+glusterd_syncop_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp);
+
+void
+gd_syncargs_init (struct syncargs *args, dict_t *op_ctx);
 #endif /* __RPC_SYNCOP_H */
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 8d81ca1af..e8ae05851 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -35,6 +35,8 @@
 #include "glusterd-store.h"
 #include "glusterd-volgen.h"
 #include "glusterd-pmap.h"
+#include "glusterfs-acl.h"
+#include "glusterd-locks.h"
 
 #include "xdr-generic.h"
 #include <sys/resource.h>
@@ -48,6 +50,11 @@
 #include <unistd.h>
 #include <fnmatch.h>
 #include <sys/statvfs.h>
+#include <ifaddrs.h>
+#ifdef HAVE_BD_XLATOR
+#include <lvm2app.h>
+#endif
+
 
 #ifdef GF_LINUX_HOST_OS
 #include <mntent.h>
@@ -68,9 +75,6 @@
 #define NLMV4_VERSION       4
 #define NLMV1_VERSION       1
 
-#define ACL_PROGRAM         100227
-#define ACLV3_VERSION       3
-
 #define CEILING_POS(X) (((X)-(int)(X)) > 0 ? (int)((X)+1) : (int)(X))
 
 static glusterd_lock_t lock;
@@ -432,8 +436,11 @@ glusterd_volinfo_new (glusterd_volinfo_t **volinfo)
         if (!new_volinfo)
                 goto out;
 
+        LOCK_INIT (&new_volinfo->lock);
         INIT_LIST_HEAD (&new_volinfo->vol_list);
+        INIT_LIST_HEAD (&new_volinfo->snapvol_list);
         INIT_LIST_HEAD (&new_volinfo->bricks);
+        INIT_LIST_HEAD (&new_volinfo->snap_volumes);
 
         new_volinfo->dict = dict_new ();
         if (!new_volinfo->dict) {
@@ -449,6 +456,10 @@ glusterd_volinfo_new (glusterd_volinfo_t **volinfo)
                 goto out;
         }
 
+        snprintf (new_volinfo->parent_volname, GLUSTERD_MAX_VOLUME_NAME, "N/A");
+
+        new_volinfo->snap_max_hard_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+
         new_volinfo->xl = THIS;
 
         *volinfo = new_volinfo;
@@ -460,6 +471,219 @@ out:
         return ret;
 }
 
+/* This function will create a new volinfo and then
+ * dup the entries from volinfo to the new_volinfo.
+ *
+ * @param volinfo       volinfo which will be duplicated
+ * @param dup_volinfo   new volinfo which will be created
+ * @param set_userauth  if this true then auth info is also set
+ *
+ * @return 0 on success else -1
+ */
+int32_t
+glusterd_volinfo_dup (glusterd_volinfo_t *volinfo,
+                      glusterd_volinfo_t **dup_volinfo,
+                      gf_boolean_t set_userauth)
+{
+        int32_t                 ret             = -1;
+        xlator_t                *this           = NULL;
+        glusterd_volinfo_t      *new_volinfo    = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
+        GF_VALIDATE_OR_GOTO (this->name, dup_volinfo, out);
+
+        ret = glusterd_volinfo_new (&new_volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "not able to create the "
+                        "duplicate volinfo for the volume %s",
+                        volinfo->volname);
+                goto out;
+        }
+
+        new_volinfo->type = volinfo->type;
+        new_volinfo->replica_count = volinfo->replica_count;
+        new_volinfo->stripe_count = volinfo->stripe_count;
+        new_volinfo->dist_leaf_count = volinfo->dist_leaf_count;
+        new_volinfo->sub_count = volinfo->sub_count;
+        new_volinfo->transport_type = volinfo->transport_type;
+        new_volinfo->nfs_transport_type = volinfo->nfs_transport_type;
+        new_volinfo->brick_count = volinfo->brick_count;
+
+        dict_copy (volinfo->dict, new_volinfo->dict);
+        gd_update_volume_op_versions (new_volinfo);
+
+        if (set_userauth) {
+                glusterd_auth_set_username (new_volinfo,
+                                            volinfo->auth.username);
+                glusterd_auth_set_password (new_volinfo,
+                                            volinfo->auth.password);
+        }
+
+        *dup_volinfo = new_volinfo;
+        ret = 0;
+out:
+        if (ret && (NULL != new_volinfo)) {
+                (void) glusterd_volinfo_delete (new_volinfo);
+        }
+        return ret;
+}
+
+/* This function will duplicate brickinfo
+ *
+ * @param brickinfo     Source brickinfo
+ * @param dup_brickinfo Destination brickinfo
+ *
+ * @return 0 on success else -1
+ */
+int32_t
+glusterd_brickinfo_dup (glusterd_brickinfo_t *brickinfo,
+                        glusterd_brickinfo_t *dup_brickinfo)
+{
+        int32_t         ret     = -1;
+        xlator_t        *this   = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_VALIDATE_OR_GOTO (this->name, brickinfo, out);
+        GF_VALIDATE_OR_GOTO (this->name, dup_brickinfo, out);
+
+        strcpy (dup_brickinfo->hostname, brickinfo->hostname);
+        strcpy (dup_brickinfo->path, brickinfo->path);
+        strcpy (dup_brickinfo->device_path, brickinfo->device_path);
+        ret = gf_canonicalize_path (dup_brickinfo->path);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to canonicalize "
+                        "brick path");
+                goto out;
+        }
+        uuid_copy (dup_brickinfo->uuid, brickinfo->uuid);
+
+        dup_brickinfo->port = brickinfo->port;
+        dup_brickinfo->rdma_port = brickinfo->rdma_port;
+        if (NULL != brickinfo->logfile) {
+                dup_brickinfo->logfile = gf_strdup (brickinfo->logfile);
+                if (NULL == dup_brickinfo->logfile) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+        dup_brickinfo->status = brickinfo->status;
+        dup_brickinfo->snap_status = brickinfo->snap_status;
+out:
+        return ret;
+}
+
+/* This function will copy snap volinfo to the new
+ * passed volinfo and regenerate backend store files
+ * for the restored snap.
+ *
+ * @param new_volinfo   new volinfo
+ * @param snap_volinfo  volinfo of snap volume
+ *
+ * @return 0 on success and -1 on failure
+ *
+ * TODO: Duplicate all members of volinfo, e.g. geo-rep sync slaves
+ */
+int32_t
+glusterd_snap_volinfo_restore (dict_t *rsp_dict,
+                               glusterd_volinfo_t *new_volinfo,
+                               glusterd_volinfo_t *snap_volinfo)
+{
+        int32_t                 brick_count     = -1;
+        int32_t                 ret             = -1;
+        xlator_t                *this           = NULL;
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+        glusterd_brickinfo_t    *new_brickinfo  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+
+        GF_VALIDATE_OR_GOTO (this->name, new_volinfo, out);
+        GF_VALIDATE_OR_GOTO (this->name, snap_volinfo, out);
+
+        brick_count = 0;
+        list_for_each_entry (brickinfo, &snap_volinfo->bricks, brick_list) {
+                ret = glusterd_brickinfo_new (&new_brickinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to create "
+                                "new brickinfo");
+                        goto out;
+                }
+
+                /* Duplicate brickinfo */
+                ret = glusterd_brickinfo_dup (brickinfo, new_brickinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to dup "
+                                "brickinfo");
+                        goto out;
+                }
+
+                /* If the brick is not of this peer, or snapshot is missed *
+                 * for the brick do not replace the xattr for it */
+                if ((!uuid_compare (brickinfo->uuid, MY_UUID)) &&
+                    (brickinfo->snap_status != -1)) {
+                        /* We need to replace the volume id of all the bricks
+                         * to the volume id of the origin volume. new_volinfo
+                         * has the origin volume's volume id*/
+                        ret = sys_lsetxattr (new_brickinfo->path,
+                                             GF_XATTR_VOL_ID_KEY,
+                                             new_volinfo->volume_id,
+                                             sizeof (new_volinfo->volume_id),
+                                             XATTR_REPLACE);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "set extended attribute %s on %s. "
+                                        "Reason: %s, snap: %s",
+                                        GF_XATTR_VOL_ID_KEY,
+                                        new_brickinfo->path, strerror (errno),
+                                        new_volinfo->volname);
+                                goto out;
+                        }
+                }
+
+                /* If a snapshot is pending for this brick then
+                 * restore should also be pending
+                 */
+                if (brickinfo->snap_status == -1) {
+                        /* Adding missed delete to the dict */
+                        ret = glusterd_add_missed_snaps_to_dict
+                                                (rsp_dict,
+                                                 snap_volinfo->volname,
+                                                 brickinfo,
+                                                 brick_count + 1,
+                                                 GF_SNAP_OPTION_TYPE_RESTORE);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to add missed snapshot info "
+                                        "for %s:%s in the rsp_dict",
+                                        brickinfo->hostname,
+                                        brickinfo->path);
+                                goto out;
+                        }
+                }
+
+                list_add_tail (&new_brickinfo->brick_list,
+                                &new_volinfo->bricks);
+                /* ownership of new_brickinfo is passed to new_volinfo */
+                new_brickinfo = NULL;
+                brick_count++;
+        }
+
+        /* Regenerate all volfiles */
+        ret = glusterd_create_volfiles_and_notify_services (new_volinfo);
+
+out:
+        if (ret && (NULL != new_brickinfo)) {
+                (void) glusterd_brickinfo_delete (new_brickinfo);
+        }
+
+        return ret;
+}
+
 void
 glusterd_auth_cleanup (glusterd_volinfo_t *volinfo) {
 
@@ -552,6 +776,7 @@ glusterd_volinfo_delete (glusterd_volinfo_t *volinfo)
         GF_ASSERT (volinfo);
 
         list_del_init (&volinfo->vol_list);
+        list_del_init (&volinfo->snapvol_list);
 
         ret = glusterd_volume_brickinfos_delete (volinfo);
         if (ret)
@@ -572,7 +797,6 @@ out:
         return ret;
 }
 
-
 int32_t
 glusterd_brickinfo_new (glusterd_brickinfo_t **brickinfo)
 {
@@ -624,6 +848,7 @@ glusterd_brickinfo_new_from_brick (char *brick,
         char                    *path = NULL;
         char                    *tmp_host = NULL;
         char                    *tmp_path = NULL;
+        char                    *vg       = NULL;
 
         GF_ASSERT (brick);
         GF_ASSERT (brickinfo);
@@ -642,6 +867,17 @@ glusterd_brickinfo_new_from_brick (char *brick,
         if (ret)
                 goto out;
 
+#ifdef HAVE_BD_XLATOR
+        vg = strchr (path, '?');
+        /* ? is used as a delimiter for vg */
+        if (vg) {
+                strncpy (new_brickinfo->vg, vg + 1, PATH_MAX - 1);
+                *vg = '\0';
+        }
+        new_brickinfo->caps = CAPS_BD;
+#else
+        vg = NULL; /* Avoid compiler warnings when BD not enabled */
+#endif
         ret = gf_canonicalize_path (path);
         if (ret)
                 goto out;
@@ -745,6 +981,62 @@ out:
         return available;
 }
 
+#ifdef HAVE_BD_XLATOR
+/*
+ * Sets the tag of the format "trusted.glusterfs.volume-id:<uuid>" in
+ * the brick VG. It is used to avoid using same VG for another brick.
+ * @volume-id - gfid, @brick - brick info, @msg - Error message returned
+ * to the caller
+ */
+int
+glusterd_bd_set_vg_tag (unsigned char *volume_id, glusterd_brickinfo_t *brick,
+                        char *msg, int msg_size)
+{
+        lvm_t        handle    = NULL;
+        vg_t         vg        = NULL;
+        char        *uuid      = NULL;
+        int          ret       = -1;
+
+        gf_asprintf (&uuid, "%s:%s", GF_XATTR_VOL_ID_KEY,
+                     uuid_utoa (volume_id));
+        if (!uuid) {
+                snprintf (msg, sizeof(*msg), "Could not allocate memory "
+                          "for tag");
+                return -1;
+        }
+
+        handle = lvm_init (NULL);
+        if (!handle) {
+                snprintf (msg, sizeof(*msg), "lvm_init failed");
+                goto out;
+        }
+
+        vg = lvm_vg_open (handle, brick->vg, "w", 0);
+        if (!vg) {
+                snprintf (msg, sizeof(*msg), "Could not open VG %s",
+                          brick->vg);
+                goto out;
+        }
+
+        if (lvm_vg_add_tag (vg, uuid) < 0) {
+                snprintf (msg, sizeof(*msg), "Could not set tag %s for "
+                          "VG %s", uuid, brick->vg);
+                goto out;
+        }
+        lvm_vg_write (vg);
+        ret = 0;
+out:
+        GF_FREE (uuid);
+
+        if (vg)
+                lvm_vg_close (vg);
+        if (handle)
+                lvm_quit (handle);
+
+        return ret;
+}
+#endif
+
 int
 glusterd_validate_and_create_brickpath (glusterd_brickinfo_t *brickinfo,
                                         uuid_t volume_id, char **op_errstr,
@@ -827,9 +1119,17 @@ glusterd_validate_and_create_brickpath (glusterd_brickinfo_t *brickinfo,
                 }
         }
 
+#ifdef HAVE_BD_XLATOR
+        if (brickinfo->vg[0]) {
+                ret = glusterd_bd_set_vg_tag (volume_id, brickinfo, msg,
+                                              sizeof(msg));
+                if (ret)
+                        goto out;
+        }
+#endif
         ret = glusterd_check_and_set_brick_xattr (brickinfo->hostname,
                                                   brickinfo->path, volume_id,
-                                                  op_errstr);
+                                                  op_errstr, is_force);
         if (ret)
                 goto out;
 
@@ -987,6 +1287,42 @@ glusterd_volinfo_find_by_volume_id (uuid_t volume_id, glusterd_volinfo_t **volin
         return ret;
 }
 
+int
+glusterd_snap_volinfo_find_by_volume_id (uuid_t volume_id,
+                                         glusterd_volinfo_t **volinfo)
+{
+        int32_t                  ret     = -1;
+        xlator_t                *this    = NULL;
+        glusterd_volinfo_t      *voliter = NULL;
+        glusterd_snap_t         *snap    = NULL;
+        glusterd_conf_t         *priv    = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (volinfo);
+
+        if (uuid_is_null(volume_id)) {
+                gf_log (this->name, GF_LOG_WARNING, "Volume UUID is NULL");
+                goto out;
+        }
+
+        list_for_each_entry (snap, &priv->snapshots, snap_list) {
+                list_for_each_entry (voliter, &snap->volumes, vol_list) {
+                        if (uuid_compare (volume_id, voliter->volume_id))
+                                continue;
+                        *volinfo = voliter;
+                        ret = 0;
+                        goto out;
+                }
+        }
+
+        gf_log (this->name, GF_LOG_WARNING, "Snap volume not found");
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
 int32_t
 glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo)
 {
@@ -996,7 +1332,6 @@ glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo)
         glusterd_conf_t         *priv = NULL;
 
         GF_ASSERT (volname);
-
         this = THIS;
         GF_ASSERT (this);
 
@@ -1005,7 +1340,8 @@ glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo)
 
         list_for_each_entry (tmp_volinfo, &priv->volumes, vol_list) {
                 if (!strcmp (tmp_volinfo->volname, volname)) {
-                        gf_log (this->name, GF_LOG_DEBUG, "Volume %s found", volname);
+                        gf_log (this->name, GF_LOG_DEBUG, "Volume %s found",
+                                volname);
                         ret = 0;
                         *volinfo = tmp_volinfo;
                         break;
@@ -1017,6 +1353,68 @@ glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo)
 }
 
 int32_t
+glusterd_snap_volinfo_find (char *snap_volname, glusterd_snap_t *snap,
+                            glusterd_volinfo_t **volinfo)
+{
+        int32_t                  ret         = -1;
+        xlator_t                *this        = NULL;
+        glusterd_volinfo_t      *snap_vol    = NULL;
+        glusterd_conf_t         *priv        = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snap);
+        GF_ASSERT (snap_volname);
+
+        list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
+                if (!strcmp (snap_vol->volname, snap_volname)) {
+                        ret = 0;
+                        *volinfo = snap_vol;
+                        goto out;
+                }
+        }
+
+        gf_log (this->name, GF_LOG_WARNING, "Snap volume %s not found",
+                snap_volname);
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_snap_volinfo_find_from_parent_volname (char *origin_volname,
+                                      glusterd_snap_t *snap,
+                                      glusterd_volinfo_t **volinfo)
+{
+        int32_t                  ret         = -1;
+        xlator_t                *this        = NULL;
+        glusterd_volinfo_t      *snap_vol    = NULL;
+        glusterd_conf_t         *priv        = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snap);
+        GF_ASSERT (origin_volname);
+
+        list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
+                if (!strcmp (snap_vol->parent_volname, origin_volname)) {
+                        ret = 0;
+                        *volinfo = snap_vol;
+                        goto out;
+                }
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Snap volume not found(snap: %s, "
+                "origin-volume: %s", snap->snapname, origin_volname);
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
 glusterd_service_stop (const char *service, char *pidfile, int sig,
                        gf_boolean_t force_kill)
 {
@@ -1096,10 +1494,9 @@ glusterd_set_brick_socket_filepath (glusterd_volinfo_t *volinfo,
  */
 int32_t
 glusterd_brick_connect (glusterd_volinfo_t  *volinfo,
-                        glusterd_brickinfo_t  *brickinfo)
+                        glusterd_brickinfo_t  *brickinfo, char *socketpath)
 {
         int                     ret = 0;
-        char                    socketpath[PATH_MAX] = {0};
         char                    volume_id_str[64];
         char                    *brickid = NULL;
         dict_t                  *options = NULL;
@@ -1108,12 +1505,9 @@ glusterd_brick_connect (glusterd_volinfo_t  *volinfo,
 
         GF_ASSERT (volinfo);
         GF_ASSERT (brickinfo);
+        GF_ASSERT (socketpath);
 
         if (brickinfo->rpc == NULL) {
-                glusterd_set_brick_socket_filepath (volinfo, brickinfo,
-                                                    socketpath,
-                                                    sizeof (socketpath));
-
                 /* Setting frame-timeout to 10mins (600seconds).
                  * Unix domain sockets ensures that the connection is reliable.
                  * The default timeout of 30mins used for unreliable network
@@ -1201,9 +1595,23 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
         priv = this->private;
         GF_ASSERT (priv);
 
+        if (brickinfo->snap_status == -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Snapshot is pending on %s:%s. "
+                        "Hence not starting the brick",
+                        brickinfo->hostname,
+                        brickinfo->path);
+                ret = 0;
+                goto out;
+        }
+
         ret = _mk_rundir_p (volinfo);
         if (ret)
                 goto out;
+
+        glusterd_set_brick_socket_filepath (volinfo, brickinfo, socketpath,
+                                            sizeof (socketpath));
+
         GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
         if (glusterd_is_service_running (pidfile, NULL))
                 goto connect;
@@ -1241,8 +1649,15 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
                 runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
         }
 
-        snprintf (volfile, PATH_MAX, "%s.%s.%s", volinfo->volname,
-                  brickinfo->hostname, exp_path);
+        if (volinfo->is_snap_volume) {
+                snprintf (volfile, PATH_MAX,"/%s/%s/%s.%s.%s",
+                          GLUSTERD_VOL_SNAP_DIR_PREFIX,
+                          volinfo->snapshot->snapname, volinfo->volname,
+                          brickinfo->hostname, exp_path);
+        } else {
+                snprintf (volfile, PATH_MAX, "%s.%s.%s", volinfo->volname,
+                          brickinfo->hostname, exp_path);
+        }
 
         if (volinfo->logdir) {
                 snprintf (logfile, PATH_MAX, "%s/%s.log",
@@ -1254,9 +1669,6 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
         if (!brickinfo->logfile)
                 brickinfo->logfile = gf_strdup (logfile);
 
-        glusterd_set_brick_socket_filepath (volinfo, brickinfo, socketpath,
-                                            sizeof (socketpath));
-
         (void) snprintf (glusterd_uuid, 1024, "*-posix.glusterd-uuid=%s",
                          uuid_utoa (MY_UUID));
         runner_add_args (&runner, SBIN_DIR"/glusterfsd",
@@ -1304,9 +1716,13 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
         brickinfo->rdma_port = rdma_port;
 
 connect:
-        ret = glusterd_brick_connect (volinfo, brickinfo);
-        if (ret)
+        ret = glusterd_brick_connect (volinfo, brickinfo, socketpath);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to connect to brick %s:%s on %s",
+                        brickinfo->hostname, brickinfo->path, socketpath);
                 goto out;
+        }
 out:
         return ret;
 }
@@ -1372,10 +1788,10 @@ glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
                                 glusterd_brickinfo_t   *brickinfo,
                                 gf_boolean_t del_brick)
 {
-        xlator_t                *this = NULL;
-        glusterd_conf_t         *priv = NULL;
-        char                    pidfile[PATH_MAX] = {0,};
-        int                     ret = 0;
+        xlator_t        *this                   = NULL;
+        glusterd_conf_t *priv                   = NULL;
+        char            pidfile[PATH_MAX]       = {0,};
+        int             ret                     = 0;
 
         GF_ASSERT (volinfo);
         GF_ASSERT (brickinfo);
@@ -1574,13 +1990,14 @@ glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo)
 
         snprintf (filepath, sizeof (filepath), "%s/%s", path,
                   GLUSTERD_VOLUME_INFO_FILE);
-        snprintf (sort_filepath, sizeof (sort_filepath), "/tmp/%s.XXXXXX",
-                  volinfo->volname);
+        snprintf (sort_filepath, sizeof (sort_filepath),
+                  "/tmp/%s.XXXXXX", volinfo->volname);
 
         sort_fd = mkstemp (sort_filepath);
         if (sort_fd < 0) {
                 gf_log (this->name, GF_LOG_ERROR, "Could not generate temp "
-                        "file, reason: %s for volume: %s", strerror (errno),
+                        "file, reason: %s for %s: %s", strerror (errno),
+                        (volinfo->is_snap_volume)?"snap":"volume",
                         volinfo->volname);
                 goto out;
         } else {
@@ -1621,7 +2038,6 @@ glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo)
                 goto out;
 
         volinfo->cksum = cksum;
-
 out:
         if (fd > 0)
                close (fd);
@@ -1700,7 +2116,10 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
         glusterd_dict_ctx_t     ctx               = {0};
         char                    *rebalance_id_str = NULL;
         char                    *rb_id_str        = NULL;
+        xlator_t                *this             = NULL;
 
+        this = THIS;
+        GF_ASSERT (this);
         GF_ASSERT (dict);
         GF_ASSERT (volinfo);
 
@@ -1715,6 +2134,15 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
         if (ret)
                 goto out;
 
+        snprintf (key, sizeof (key), "volume%d.is_volume_restored", count);
+        ret = dict_set_int32 (dict, key, volinfo->is_volume_restored);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set "
+                        "is_volume_restored option for %s volume",
+                        volinfo->volname);
+                goto out;
+        }
+
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.brick_count", count);
         ret = dict_set_int32 (dict, key, volinfo->brick_count);
@@ -1769,6 +2197,20 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
         if (ret)
                 goto out;
 
+        snprintf (key, sizeof (key), "volume%d.is_snap_volume", count);
+        ret = dict_set_uint32 (dict, key, volinfo->is_snap_volume);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Unable to set %s", key);
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "volume%d.snap-max-hard-limit", count);
+        ret = dict_set_uint64 (dict, key, volinfo->snap_max_hard_limit);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Unable to set %s", key);
+                goto out;
+        }
+
         volume_id_str = gf_strdup (uuid_utoa (volinfo->volume_id));
         if (!volume_id_str) {
                 ret = -1;
@@ -1915,9 +2357,42 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                 if (ret)
                         goto out;
 
+                snprintf (key, sizeof (key), "volume%d.brick%d.snap_status",
+                          count, i);
+                ret = dict_set_int32 (dict, key, brickinfo->snap_status);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set snap_status for %s:%s",
+                                brickinfo->hostname,
+                                brickinfo->path);
+                        goto out;
+                }
+
+                snprintf (key, sizeof (key), "volume%d.brick%d.device_path",
+                          count, i);
+                ret = dict_set_str (dict, key, brickinfo->device_path);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set snap_device for %s:%s",
+                                brickinfo->hostname,
+                                brickinfo->path);
+                        goto out;
+                }
+
                 i++;
         }
 
+        /* Add volume op-versions to dict. This prevents volume inconsistencies
+         * in the cluster
+         */
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.op-version", count);
+        ret = dict_set_int32 (dict, key, volinfo->op_version);
+        if (ret)
+                goto out;
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.client-op-version", count);
+        ret = dict_set_int32 (dict, key, volinfo->client_op_version);
 
 out:
         GF_FREE (volume_id_str);
@@ -2304,7 +2779,7 @@ glusterd_do_volume_quorum_action (xlator_t *this, glusterd_volinfo_t *volinfo,
                                   gf_boolean_t meets_quorum)
 {
         glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_conf_t         *conf = NULL;
+        glusterd_conf_t         *conf      = NULL;
 
         conf = this->private;
         if (volinfo->status != GLUSTERD_STATUS_STARTED)
@@ -2422,6 +2897,8 @@ glusterd_import_new_brick (dict_t *vols, int32_t vol_count,
 {
         char                    key[512] = {0,};
         int                     ret = -1;
+        int32_t                 snap_status = 0;
+        char                    *snap_device = NULL;
         char                    *hostname = NULL;
         char                    *path = NULL;
         glusterd_brickinfo_t    *new_brickinfo = NULL;
@@ -2449,12 +2926,31 @@ glusterd_import_new_brick (dict_t *vols, int32_t vol_count,
                 goto out;
         }
 
+        snprintf (key, sizeof (key), "volume%d.brick%d.snap_status",
+                  vol_count, brick_count);
+        ret = dict_get_int32 (vols, key, &snap_status);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload", key);
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "volume%d.brick%d.device_path",
+                  vol_count, brick_count);
+        ret = dict_get_str (vols, key, &snap_device);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload", key);
+                goto out;
+        }
+
         ret = glusterd_brickinfo_new (&new_brickinfo);
         if (ret)
                 goto out;
 
         strcpy (new_brickinfo->path, path);
         strcpy (new_brickinfo->hostname, hostname);
+        strcpy (new_brickinfo->device_path, snap_device);
+        new_brickinfo->snap_status = snap_status;
+
         //peerinfo might not be added yet
         (void) glusterd_resolve_brick (new_brickinfo);
         ret = 0;
@@ -2508,6 +3004,9 @@ glusterd_import_volinfo (dict_t *vols, int count,
         int                rb_status         = 0;
         char               *rebalance_id_str = NULL;
         char               *rb_id_str        = NULL;
+        int                op_version        = 0;
+        int                client_op_version = 0;
+        uint32_t           is_snap_volume    = 0;
 
         GF_ASSERT (vols);
         GF_ASSERT (volinfo);
@@ -2519,6 +3018,22 @@ glusterd_import_volinfo (dict_t *vols, int count,
                 goto out;
         }
 
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.is_snap_volume", count);
+        ret = dict_get_uint32 (vols, key, &is_snap_volume);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
+        if (is_snap_volume == _gf_true) {
+                gf_log (THIS->name, GF_LOG_DEBUG,
+                        "Not syncing snap volume %s", volname);
+                ret = 0;
+                goto out;
+        }
+
         ret = glusterd_volinfo_new (&new_volinfo);
         if (ret)
                 goto out;
@@ -2644,6 +3159,25 @@ glusterd_import_volinfo (dict_t *vols, int count,
                 goto out;
         }
 
+        new_volinfo->is_snap_volume = is_snap_volume;
+
+        snprintf (key, sizeof (key), "volume%d.is_volume_restored", count);
+        ret = dict_get_uint32 (vols, key, &new_volinfo->is_volume_restored);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get "
+                        "is_volume_restored option for %s",
+                        volname);
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "volume%d.snap-max-hard-limit", count);
+        ret = dict_get_uint64 (vols, key, &new_volinfo->snap_max_hard_limit);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.rebalance", count);
         ret = dict_get_uint32 (vols, key, &new_volinfo->rebal.defrag_cmd);
@@ -2734,6 +3268,40 @@ glusterd_import_volinfo (dict_t *vols, int count,
         ret = glusterd_import_friend_volume_opts (vols, count, new_volinfo);
         if (ret)
                 goto out;
+
+        /* Import the volume's op-versions if available else set it to 1.
+         * Not having op-versions implies this informtation was obtained from a
+         * op-version 1 friend (gluster-3.3), ergo the cluster is at op-version
+         * 1 and all volumes are at op-versions 1.
+         *
+         * Either both the volume op-versions should be absent or both should be
+         * present. Only one being present is a failure
+         */
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.op-version", count);
+        ret = dict_get_int32 (vols, key, &op_version);
+        if (ret)
+                ret = 0;
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.client-op-version", count);
+        ret = dict_get_int32 (vols, key, &client_op_version);
+        if (ret)
+                ret = 0;
+
+        if (op_version && client_op_version) {
+                new_volinfo->op_version = op_version;
+                new_volinfo->client_op_version = client_op_version;
+        } else if (((op_version == 0) && (client_op_version != 0)) ||
+                   ((op_version != 0) && (client_op_version == 0))) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR,
+                        "Only one volume op-version found");
+                goto out;
+        } else {
+                new_volinfo->op_version = 1;
+                new_volinfo->client_op_version = 1;
+        }
+
         ret = glusterd_import_bricks (vols, count, new_volinfo);
         if (ret)
                 goto out;
@@ -2892,6 +3460,12 @@ glusterd_import_friend_volume (dict_t *vols, size_t count)
         if (ret)
                 goto out;
 
+        if (!new_volinfo) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Not importing snap volume");
+                goto out;
+        }
+
         ret = glusterd_volinfo_find (new_volinfo->volname, &old_volinfo);
         if (0 == ret) {
                 (void) glusterd_delete_stale_volume (old_volinfo, new_volinfo);
@@ -2901,8 +3475,6 @@ glusterd_import_friend_volume (dict_t *vols, size_t count)
                 (void) glusterd_start_bricks (new_volinfo);
         }
 
-        gd_update_volume_op_versions (new_volinfo);
-
         ret = glusterd_store_volinfo (new_volinfo, GLUSTERD_VOLINFO_VER_AC_NONE);
         ret = glusterd_create_volfiles_and_notify_services (new_volinfo);
         if (ret)
@@ -3710,6 +4282,11 @@ glusterd_reconfigure_nfs ()
         int             ret             = -1;
         gf_boolean_t    identical       = _gf_false;
 
+        /*
+         * Check both OLD and NEW volfiles, if they are SAME by size
+         * and cksum i.e. "character-by-character". If YES, then
+         * NOTHING has been changed, just return.
+         */
         ret = glusterd_check_nfs_volfile_identical (&identical);
         if (ret)
                 goto out;
@@ -3719,6 +4296,31 @@ glusterd_reconfigure_nfs ()
                 goto out;
         }
 
+        /*
+         * They are not identical. Find out if the topology is changed
+         * OR just the volume options. If just the options which got
+         * changed, then inform the xlator to reconfigure the options.
+         */
+        identical = _gf_false; /* RESET the FLAG */
+        ret = glusterd_check_nfs_topology_identical (&identical);
+        if (ret)
+                goto out;
+
+        /* Topology is not changed, but just the options. But write the
+         * options to NFS volfile, so that NFS will be reconfigured.
+         */
+        if (identical) {
+                ret = glusterd_create_nfs_volfile();
+                if (ret == 0) {/* Only if above PASSES */
+                        ret = glusterd_fetchspec_notify (THIS);
+                }
+                goto out;
+        }
+
+        /*
+         * NFS volfile's topology has been changed. NFS server needs
+         * to be RESTARTED to ACT on the changed volfile.
+         */
         ret = glusterd_check_generate_start_nfs ();
 
 out:
@@ -3950,20 +4552,42 @@ out:
 int
 glusterd_restart_bricks (glusterd_conf_t *conf)
 {
+        int                   ret            = 0;
         glusterd_volinfo_t   *volinfo        = NULL;
         glusterd_brickinfo_t *brickinfo      = NULL;
+        glusterd_snap_t      *snap           = NULL;
         gf_boolean_t          start_nodesvcs = _gf_false;
-        int                   ret            = 0;
+        xlator_t             *this           = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
 
         list_for_each_entry (volinfo, &conf->volumes, vol_list) {
                 if (volinfo->status != GLUSTERD_STATUS_STARTED)
                         continue;
                 start_nodesvcs = _gf_true;
+                gf_log (this->name, GF_LOG_DEBUG, "starting the volume %s",
+                        volinfo->volname);
                 list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
                         glusterd_brick_start (volinfo, brickinfo, _gf_false);
                 }
         }
 
+        list_for_each_entry (snap, &conf->snapshots, snap_list) {
+                list_for_each_entry (volinfo, &snap->volumes, vol_list) {
+                        if (volinfo->status != GLUSTERD_STATUS_STARTED)
+                                continue;
+                        start_nodesvcs = _gf_true;
+                        gf_log (this->name, GF_LOG_DEBUG, "starting the snap "
+                                "volume %s", volinfo->volname);
+                        list_for_each_entry (brickinfo, &volinfo->bricks,
+                                             brick_list) {
+                                glusterd_brick_start (volinfo, brickinfo,
+                                                      _gf_false);
+                        }
+                }
+        }
+
         if (start_nodesvcs)
                 glusterd_nodesvcs_handle_graph_change (NULL);
 
@@ -4129,7 +4753,7 @@ out:
 }
 
 #ifdef GF_LINUX_HOST_OS
-static int
+int
 glusterd_get_brick_root (char *path, char **mount_point)
 {
         char           *ptr            = NULL;
@@ -4297,6 +4921,31 @@ glusterd_add_inode_size_to_dict (dict_t *dict, int count)
         return ret;
 }
 
+struct mntent *
+glusterd_get_mnt_entry_info (char *mnt_pt, FILE *mtab)
+{
+        struct mntent  *entry                = NULL;
+
+        mtab = setmntent (_PATH_MOUNTED, "r");
+        if (!mtab)
+                goto out;
+
+        entry = getmntent (mtab);
+
+        while (1) {
+                if (!entry)
+                        goto out;
+
+                if (!strcmp (entry->mnt_dir, mnt_pt) &&
+                    strcmp (entry->mnt_type, "rootfs"))
+                        break;
+                entry = getmntent (mtab);
+        }
+
+out:
+        return entry;
+}
+
 static int
 glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo,
                                   dict_t *dict, int count)
@@ -4308,8 +4957,8 @@ glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo,
         char           *fs_name              = NULL;
         char           *mnt_options          = NULL;
         char           *device               = NULL;
-        FILE           *mtab                 = NULL;
         struct mntent  *entry                = NULL;
+        FILE           *mtab                 = NULL;
 
         snprintf (base_key, sizeof (base_key), "brick%d", count);
 
@@ -4317,25 +4966,12 @@ glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo,
         if (ret)
                 goto out;
 
-        mtab = setmntent (_PATH_MOUNTED, "r");
-        if (!mtab) {
+        entry = glusterd_get_mnt_entry_info (mnt_pt, mtab);
+        if (!entry) {
                 ret = -1;
                 goto out;
         }
 
-        entry = getmntent (mtab);
-
-        while (1) {
-                if (!entry) {
-                        ret = -1;
-                        goto out;
-                }
-                if (!strcmp (entry->mnt_dir, mnt_pt) &&
-                    strcmp (entry->mnt_type, "rootfs"))
-                        break;
-                entry = getmntent (mtab);
-        }
-
         /* get device file */
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "%s.device", base_key);
@@ -4368,6 +5004,45 @@ glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo,
 
         return ret;
 }
+
+char*
+glusterd_get_brick_mount_details (glusterd_brickinfo_t *brickinfo)
+{
+        int             ret                  = -1;
+        char           *mnt_pt               = NULL;
+        char           *device               = NULL;
+        FILE           *mtab                 = NULL;
+        struct mntent  *entry                = NULL;
+        xlator_t       *this                 = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (brickinfo);
+
+        ret = glusterd_get_brick_root (brickinfo->path, &mnt_pt);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get mount point "
+                        "for %s brick", brickinfo->path);
+                goto out;
+        }
+
+        entry = glusterd_get_mnt_entry_info (mnt_pt, mtab);
+        if (NULL == entry) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get mnt entry "
+                        "for %s mount path", mnt_pt);
+                goto out;
+        }
+
+        /* get the fs_name/device */
+        device = gf_strdup (entry->mnt_fsname);
+
+out:
+        if (NULL != mtab) {
+                endmntent (mtab);
+        }
+
+        return device;
+}
 #endif
 
 int
@@ -4728,6 +5403,7 @@ glusterd_hostname_to_uuid (char *hostname, uuid_t uuid)
                         uuid_copy (uuid, MY_UUID);
                         ret = 0;
                 } else {
+                        ret = 0;
                         goto out;
                 }
         } else {
@@ -5062,11 +5738,12 @@ out:
 
 int
 glusterd_check_and_set_brick_xattr (char *host, char *path, uuid_t uuid,
-                                    char **op_errstr)
+                                    char **op_errstr, gf_boolean_t is_force)
 {
         int             ret             = -1;
         char            msg[2048]       = {0,};
         gf_boolean_t    in_use          = _gf_false;
+        int             flags           = 0;
 
         /* Check for xattr support in backend fs */
         ret = sys_lsetxattr (path, "trusted.glusterfs.test",
@@ -5086,13 +5763,17 @@ glusterd_check_and_set_brick_xattr (char *host, char *path, uuid_t uuid,
         if (ret)
                 goto out;
 
-        if (in_use) {
+        if (in_use && !is_force) {
                 ret = -1;
                 goto out;
         }
 
+
+        if (!is_force)
+                flags = XATTR_CREATE;
+
         ret = sys_lsetxattr (path, GF_XATTR_VOL_ID_KEY, uuid, 16,
-                             XATTR_CREATE);
+                             flags);
         if (ret) {
                 snprintf (msg, sizeof (msg), "Failed to set extended "
                           "attributes %s, reason: %s",
@@ -6236,6 +6917,68 @@ glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
 }
 
 int
+glusterd_check_topology_identical (const char *filename1,
+                                   const char *filename2,
+                                   gf_boolean_t *identical)
+{
+        int                     ret = -1; /* FAILURE */
+        xlator_t                *this = NULL;
+        FILE                    *fp1 = NULL;
+        FILE                    *fp2 = NULL;
+        glusterfs_graph_t       *grph1 = NULL;
+        glusterfs_graph_t       *grph2 = NULL;
+
+        if ((!filename1) || (!filename2) || (!identical))
+                goto out;
+
+        this = THIS;
+
+        errno = 0; /* RESET the errno */
+
+        /* fopen() the volfile1 to create the graph */
+        fp1 = fopen (filename1, "r");
+        if (fp1 == NULL) {
+                gf_log (this->name, GF_LOG_ERROR, "fopen() on file: %s failed "
+                        "(%s)", filename1, strerror (errno));
+                goto out;
+        }
+
+        /* fopen() the volfile2 to create the graph */
+        fp2 = fopen (filename2, "r");
+        if (fp2 == NULL) {
+                gf_log (this->name, GF_LOG_ERROR, "fopen() on file: %s failed "
+                        "(%s)", filename2, strerror (errno));
+                goto out;
+        }
+
+        /* create the graph for filename1 */
+        grph1 = glusterfs_graph_construct(fp1);
+        if (grph1 == NULL)
+                goto out;
+
+        /* create the graph for filename2 */
+        grph2 = glusterfs_graph_construct(fp2);
+        if (grph2 == NULL)
+                goto out;
+
+        /* compare the graph topology */
+        *identical = is_graph_topology_equal(grph1, grph2);
+        ret = 0; /* SUCCESS */
+out:
+        if (fp1)
+                fclose(fp1);
+        if (fp2)
+                fclose(fp2);
+        if (grph1)
+                glusterfs_graph_destroy(grph1);
+        if (grph2)
+                glusterfs_graph_destroy(grph2);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int
 glusterd_check_files_identical (char *filename1, char *filename2,
                                 gf_boolean_t *identical)
 {
@@ -6748,7 +7491,7 @@ glusterd_volume_status_copy_to_op_ctx_dict (dict_t *aggr, dict_t *rsp_dict)
         if (ret)
                 goto out;
 
-        if (cmd & GF_CLI_STATUS_ALL && is_origin_glusterd ()) {
+        if (cmd & GF_CLI_STATUS_ALL && is_origin_glusterd (ctx_dict)) {
                 ret = dict_get_int32 (rsp_dict, "vol_count", &vol_count);
                 if (ret == 0) {
                         ret = dict_set_int32 (ctx_dict, "vol_count",
@@ -6770,6 +7513,12 @@ glusterd_volume_status_copy_to_op_ctx_dict (dict_t *aggr, dict_t *rsp_dict)
                 }
         }
 
+        if ((cmd & GF_CLI_STATUS_TASKS) != 0) {
+                dict_copy (rsp_dict, aggr);
+                ret = 0;
+                goto out;
+        }
+
         ret = dict_get_int32 (rsp_dict, "count", &rsp_node_count);
         if (ret) {
                 ret = 0; //no bricks in the rsp
@@ -7003,6 +7752,279 @@ out:
 }
 
 int
+glusterd_snap_config_use_rsp_dict (dict_t *dst, dict_t *src)
+{
+        char           buf[PATH_MAX]        = "";
+        char          *volname              = NULL;
+        int            ret                  = -1;
+        int            config_command       = 0;
+        uint64_t       i                    = 0;
+        uint64_t       value                = 0;
+        uint64_t       voldisplaycount      = 0;
+
+        if (!dst || !src) {
+                gf_log ("", GF_LOG_ERROR, "Source or Destination "
+                        "dict is empty.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dst, "config-command", &config_command);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "failed to get config-command type");
+                goto out;
+        }
+
+        switch (config_command) {
+        case GF_SNAP_CONFIG_DISPLAY:
+                ret = dict_get_uint64 (src, "snap-max-hard-limit", &value);
+                if (!ret) {
+                        ret = dict_set_uint64 (dst, "snap-max-hard-limit", value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to set snap_max_hard_limit");
+                                goto out;
+                        }
+                } else {
+                        /* Received dummy response from other nodes */
+                        ret = 0;
+                        goto out;
+                }
+
+                ret = dict_get_uint64 (src, "snap-max-soft-limit", &value);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to get snap_max_soft_limit");
+                        goto out;
+                }
+
+                ret = dict_set_uint64 (dst, "snap-max-soft-limit", value);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to set snap_max_soft_limit");
+                        goto out;
+                }
+
+                ret = dict_get_uint64 (src, "voldisplaycount",
+                                       &voldisplaycount);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to get voldisplaycount");
+                        goto out;
+                }
+
+                ret = dict_set_uint64 (dst, "voldisplaycount",
+                                       voldisplaycount);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to set voldisplaycount");
+                        goto out;
+                }
+
+                for (i = 0; i < voldisplaycount; i++) {
+                        snprintf (buf, sizeof(buf), "volume%ld-volname", i);
+                        ret = dict_get_str (src, buf, &volname);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to get %s", buf);
+                                goto out;
+                        }
+                        ret = dict_set_str (dst, buf, volname);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-snap-max-hard-limit", i);
+                        ret = dict_get_uint64 (src, buf, &value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to get %s", buf);
+                                goto out;
+                        }
+                        ret = dict_set_uint64 (dst, buf, value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-active-hard-limit", i);
+                        ret = dict_get_uint64 (src, buf, &value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to get %s", buf);
+                                goto out;
+                        }
+                        ret = dict_set_uint64 (dst, buf, value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-snap-max-soft-limit", i);
+                        ret = dict_get_uint64 (src, buf, &value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to get %s", buf);
+                                goto out;
+                        }
+                        ret = dict_set_uint64 (dst, buf, value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to set %s", buf);
+                                goto out;
+                        }
+                }
+
+                break;
+        default:
+                break;
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+/* Aggregate missed_snap_counts from different nodes and save it *
+ * in the req_dict of the originator node */
+int
+glusterd_snap_create_use_rsp_dict (dict_t *dst, dict_t *src)
+{
+        char          *buf                      = NULL;
+        char          *tmp_str                  = NULL;
+        char           name_buf[PATH_MAX]       = "";
+        int32_t        i                        = -1;
+        int32_t        ret                      = -1;
+        int32_t        src_missed_snap_count    = -1;
+        int32_t        dst_missed_snap_count    = -1;
+        xlator_t      *this                     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (!dst || !src) {
+                gf_log (this->name, GF_LOG_ERROR, "Source or Destination "
+                        "dict is empty.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (src, "missed_snap_count",
+                              &src_missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "No missed snaps");
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_get_int32 (dst, "missed_snap_count",
+                              &dst_missed_snap_count);
+        if (ret) {
+                /* Initialize dst_missed_count for the first time */
+                dst_missed_snap_count = 0;
+        }
+
+        for (i = 0; i < src_missed_snap_count; i++) {
+                 snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
+                           i);
+                 ret = dict_get_str (src, name_buf, &buf);
+                 if (ret) {
+                         gf_log (this->name, GF_LOG_ERROR,
+                                 "Unable to fetch %s", name_buf);
+                         goto out;
+                 }
+
+                 snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
+                           dst_missed_snap_count);
+
+                 tmp_str = gf_strdup (buf);
+                 if (!tmp_str) {
+                         ret = -1;
+                         goto out;
+                 }
+
+                 ret = dict_set_dynstr (dst, name_buf, tmp_str);
+                 if (ret) {
+                         gf_log (this->name, GF_LOG_ERROR,
+                                 "Unable to set %s", name_buf);
+                         goto out;
+                 }
+
+                 tmp_str = NULL;
+                 dst_missed_snap_count++;
+        }
+
+        ret = dict_set_int32 (dst, "missed_snap_count", dst_missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set dst_missed_snap_count");
+                goto out;
+        }
+
+out:
+        if (ret && tmp_str)
+                GF_FREE(tmp_str);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_snap_use_rsp_dict (dict_t *dst, dict_t *src)
+{
+        int            ret            = -1;
+        int32_t        snap_command   = 0;
+
+        if (!dst || !src) {
+                gf_log ("", GF_LOG_ERROR, "Source or Destination "
+                        "dict is empty.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dst, "type", &snap_command);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                ret = glusterd_snap_create_use_rsp_dict (dst, src);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to use rsp dict");
+                        goto out;
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+                ret = glusterd_snap_config_use_rsp_dict (dst, src);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to use rsp dict");
+                        goto out;
+                }
+                break;
+        default:
+                // copy the response dictinary's contents to the dict to be
+                // sent back to the cli
+                dict_copy (src, dst);
+                break;
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
 glusterd_sys_exec_output_rsp_dict (dict_t *dst, dict_t *src)
 {
         char           output_name[PATH_MAX] = "";
@@ -7230,6 +8252,77 @@ out:
 }
 
 int
+_heal_volume_add_shd_rsp_of_statistics (dict_t *this, char *key, data_t
+                                             *value, void *data)
+{
+        char                            new_key[256] = {0,};
+        char                            int_str[16] = {0,};
+        char                            key_begin_string[128] = {0,};
+        data_t                          *new_value = NULL;
+        char                            *rxl_end = NULL;
+        char                            *rxl_child_end = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        char                            *key_begin_str = NULL;
+        int                             rxl_id = 0;
+        int                             rxl_child_id = 0;
+        int                             brick_id = 0;
+        int                             int_len = 0;
+        int                             ret = 0;
+        glusterd_heal_rsp_conv_t        *rsp_ctx = NULL;
+        glusterd_brickinfo_t            *brickinfo = NULL;
+
+        rsp_ctx = data;
+        key_begin_str = strchr (key, '-');
+        if (!key_begin_str)
+                goto out;
+
+        int_len = strlen (key) - strlen (key_begin_str);
+        strncpy (key_begin_string, key, int_len);
+        key_begin_string[int_len] = '\0';
+
+        rxl_end = strchr (key_begin_str + 1, '-');
+        if (!rxl_end)
+                goto out;
+
+        int_len = strlen (key_begin_str) - strlen (rxl_end) - 1;
+        strncpy (int_str, key_begin_str + 1, int_len);
+        int_str[int_len] = '\0';
+        ret = gf_string2int (int_str, &rxl_id);
+        if (ret)
+                goto out;
+
+
+        rxl_child_end = strchr (rxl_end + 1, '-');
+        if (!rxl_child_end)
+                goto out;
+
+        int_len = strlen (rxl_end) - strlen (rxl_child_end) - 1;
+        strncpy (int_str, rxl_end + 1, int_len);
+        int_str[int_len] = '\0';
+        ret = gf_string2int (int_str, &rxl_child_id);
+        if (ret)
+                goto out;
+
+        volinfo = rsp_ctx->volinfo;
+        brick_id = rxl_id * volinfo->replica_count + rxl_child_id;
+
+        brickinfo = glusterd_get_brickinfo_by_position (volinfo, brick_id);
+        if (!brickinfo)
+                goto out;
+        if (!glusterd_is_local_brick (rsp_ctx->this, volinfo, brickinfo))
+                goto out;
+
+        new_value = data_copy (value);
+        snprintf (new_key, sizeof (new_key), "%s-%d%s", key_begin_string,
+                  brick_id, rxl_child_end);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+
+out:
+        return 0;
+
+}
+
+int
 glusterd_heal_volume_brick_rsp (dict_t *req_dict, dict_t *rsp_dict,
                                 dict_t *op_ctx, char **op_errstr)
 {
@@ -7237,6 +8330,7 @@ glusterd_heal_volume_brick_rsp (dict_t *req_dict, dict_t *rsp_dict,
         glusterd_heal_rsp_conv_t        rsp_ctx = {0};
         char                            *volname = NULL;
         glusterd_volinfo_t              *volinfo = NULL;
+        int                             heal_op = -1;
 
         GF_ASSERT (rsp_dict);
         GF_ASSERT (op_ctx);
@@ -7248,6 +8342,13 @@ glusterd_heal_volume_brick_rsp (dict_t *req_dict, dict_t *rsp_dict,
                 goto out;
         }
 
+        ret = dict_get_int32 (req_dict, "heal-op", &heal_op);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get heal_op");
+                goto out;
+        }
+
+
         ret  = glusterd_volinfo_find (volname, &volinfo);
 
         if (ret)
@@ -7256,7 +8357,12 @@ glusterd_heal_volume_brick_rsp (dict_t *req_dict, dict_t *rsp_dict,
         rsp_ctx.dict = op_ctx;
         rsp_ctx.volinfo = volinfo;
         rsp_ctx.this = THIS;
-        dict_foreach (rsp_dict, _heal_volume_add_shd_rsp, &rsp_ctx);
+        if (heal_op == GF_AFR_OP_STATISTICS)
+                dict_foreach (rsp_dict, _heal_volume_add_shd_rsp_of_statistics,
+                              &rsp_ctx);
+        else
+                dict_foreach (rsp_dict, _heal_volume_add_shd_rsp, &rsp_ctx);
+
 
 out:
         return ret;
@@ -7459,16 +8565,31 @@ glusterd_handle_node_rsp (dict_t *req_dict, void *pending_entry,
  * time a lock_owner is set
  */
 gf_boolean_t
-is_origin_glusterd ()
+is_origin_glusterd (dict_t *dict)
 {
-        int     ret = 0;
-        uuid_t  lock_owner = {0,};
+        gf_boolean_t  ret              = _gf_false;
+        uuid_t        lock_owner       = {0,};
+        uuid_t        *originator_uuid = NULL;
 
-        ret = glusterd_get_lock_owner (&lock_owner);
-        if (ret)
-                return _gf_false;
+        GF_ASSERT (dict);
+
+        ret = dict_get_bin (dict, "originator_uuid",
+                            (void **) &originator_uuid);
+        if (ret) {
+                /* If not originator_uuid has been set, then the command
+                 * has been originated from a glusterd running on older version
+                 * Hence fetching the lock owner */
+                ret = glusterd_get_lock_owner (&lock_owner);
+                if (ret) {
+                        ret = _gf_false;
+                        goto out;
+                }
+                ret = !uuid_compare (MY_UUID, lock_owner);
+        } else
+                ret = !uuid_compare (MY_UUID, *originator_uuid);
 
-        return (uuid_compare (MY_UUID, lock_owner) == 0);
+out:
+        return ret;
 }
 
 int
@@ -7652,3 +8773,122 @@ out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
+
+gf_boolean_t
+glusterd_is_status_tasks_op (glusterd_op_t op, dict_t *dict)
+{
+        int           ret             = -1;
+        uint32_t      cmd             = GF_CLI_STATUS_NONE;
+        gf_boolean_t  is_status_tasks = _gf_false;
+
+        if (op != GD_OP_STATUS_VOLUME)
+                goto out;
+
+        ret = dict_get_uint32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get opcode");
+                goto out;
+        }
+
+        if (cmd & GF_CLI_STATUS_TASKS)
+                is_status_tasks = _gf_true;
+
+out:
+        return is_status_tasks;
+}
+
+int
+glusterd_compare_snap_time(struct list_head *list1, struct list_head *list2)
+{
+        glusterd_snap_t *snap1 = NULL;
+        glusterd_snap_t *snap2 = NULL;
+        double diff_time       = 0;
+
+        GF_ASSERT (list1);
+        GF_ASSERT (list2);
+
+        snap1 = list_entry(list1, glusterd_snap_t, snap_list);
+        snap2 = list_entry(list2, glusterd_snap_t, snap_list);
+        diff_time = difftime(snap1->time_stamp, snap2->time_stamp);
+
+        return ((int)diff_time);
+}
+
+int
+glusterd_compare_snap_vol_time(struct list_head *list1, struct list_head *list2)
+{
+        glusterd_volinfo_t *snapvol1 = NULL;
+        glusterd_volinfo_t *snapvol2 = NULL;
+        double diff_time             = 0;
+
+        GF_ASSERT (list1);
+        GF_ASSERT (list2);
+
+        snapvol1 = list_entry(list1, glusterd_volinfo_t, snapvol_list);
+        snapvol2 = list_entry(list2, glusterd_volinfo_t, snapvol_list);
+        diff_time = difftime(snapvol1->snapshot->time_stamp,
+                             snapvol2->snapshot->time_stamp);
+
+        return ((int)diff_time);
+}
+
+int32_t
+glusterd_missed_snapinfo_new (glusterd_missed_snap_info **missed_snapinfo)
+{
+        glusterd_missed_snap_info      *new_missed_snapinfo = NULL;
+        int32_t                         ret                 = -1;
+        xlator_t                       *this                = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (missed_snapinfo);
+
+        new_missed_snapinfo = GF_CALLOC (1, sizeof(*new_missed_snapinfo),
+                                         gf_gld_mt_missed_snapinfo_t);
+
+        if (!new_missed_snapinfo)
+                goto out;
+
+        new_missed_snapinfo->node_snap_info = NULL;
+        INIT_LIST_HEAD (&new_missed_snapinfo->missed_snaps);
+        INIT_LIST_HEAD (&new_missed_snapinfo->snap_ops);
+
+        *missed_snapinfo = new_missed_snapinfo;
+
+        ret = 0;
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_missed_snap_op_new (glusterd_snap_op_t **snap_op)
+{
+        glusterd_snap_op_t      *new_snap_op = NULL;
+        int32_t                  ret         = -1;
+        xlator_t                *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (snap_op);
+
+        new_snap_op = GF_CALLOC (1, sizeof(*new_snap_op),
+                                 gf_gld_mt_missed_snapinfo_t);
+
+        if (!new_snap_op)
+                goto out;
+
+        new_snap_op->brick_path = NULL;
+        new_snap_op->brick_num = -1;
+        new_snap_op->op = -1;
+        new_snap_op->status = -1;
+        INIT_LIST_HEAD (&new_snap_op->snap_ops_list);
+
+        *snap_op = new_snap_op;
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 89888b419..56bb799bf 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -81,6 +81,11 @@ glusterd_submit_request (struct rpc_clnt *rpc, void *req,
 int32_t
 glusterd_volinfo_new (glusterd_volinfo_t **volinfo);
 
+int32_t
+glusterd_volinfo_dup (glusterd_volinfo_t *volinfo,
+                      glusterd_volinfo_t **dup_volinfo,
+                      gf_boolean_t set_userauth);
+
 char *
 glusterd_auth_get_username (glusterd_volinfo_t *volinfo);
 
@@ -115,11 +120,23 @@ int32_t
 glusterd_peer_hostname_new (char *hostname, glusterd_peer_hostname_t **name);
 
 int32_t
+glusterd_snap_volinfo_find (char *volname, glusterd_snap_t *snap,
+                            glusterd_volinfo_t **volinfo);
+int32_t
+glusterd_snap_volinfo_find_from_parent_volname (char *origin_volname,
+                                      glusterd_snap_t *snap,
+                                      glusterd_volinfo_t **volinfo);
+
+int32_t
 glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo);
 
 int
 glusterd_volinfo_find_by_volume_id (uuid_t volume_id, glusterd_volinfo_t **volinfo);
 
+int
+glusterd_snap_volinfo_find_by_volume_id (uuid_t volume_id,
+                                         glusterd_volinfo_t **volinfo);
+
 int32_t
 glusterd_service_stop(const char *service, char *pidfile, int sig,
                       gf_boolean_t force_kill);
@@ -282,6 +299,7 @@ glusterd_is_defrag_on (glusterd_volinfo_t *volinfo);
 
 int32_t
 glusterd_volinfo_bricks_delete (glusterd_volinfo_t *volinfo);
+
 int
 glusterd_friend_find_by_uuid (uuid_t uuid,
                               glusterd_peerinfo_t  **peerinfo);
@@ -318,7 +336,7 @@ glusterd_rb_check_bricks (glusterd_volinfo_t *volinfo,
 
 int
 glusterd_check_and_set_brick_xattr (char *host, char *path, uuid_t uuid,
-                                    char **op_errstr);
+                                    char **op_errstr, gf_boolean_t is_force);
 
 int
 glusterd_validate_and_create_brickpath (glusterd_brickinfo_t *brickinfo,
@@ -351,7 +369,7 @@ gf_boolean_t
 glusterd_peerinfo_is_uuid_unknown (glusterd_peerinfo_t *peerinfo);
 int32_t
 glusterd_brick_connect (glusterd_volinfo_t  *volinfo,
-                        glusterd_brickinfo_t  *brickinfo);
+                        glusterd_brickinfo_t  *brickinfo, char *socketpath);
 int32_t
 glusterd_brick_disconnect (glusterd_brickinfo_t *brickinfo);
 int32_t
@@ -359,12 +377,16 @@ glusterd_delete_volume (glusterd_volinfo_t *volinfo);
 int32_t
 glusterd_delete_brick (glusterd_volinfo_t* volinfo,
                        glusterd_brickinfo_t *brickinfo);
+
 int32_t
 glusterd_delete_all_bricks (glusterd_volinfo_t* volinfo);
+
 int
 glusterd_spawn_daemons (void *opaque);
+
 int
 glusterd_restart_gsyncds (glusterd_conf_t *conf);
+
 int
 glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
                       char *path_list, char *conf_path,
@@ -456,6 +478,12 @@ glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
 int
 glusterd_check_files_identical (char *filename1, char *filename2,
                                 gf_boolean_t *identical);
+
+int
+glusterd_check_topology_identical (const char *filename1,
+                                   const char *filename2,
+                                   gf_boolean_t *identical);
+
 void
 glusterd_volinfo_reset_defrag_stats (glusterd_volinfo_t *volinfo);
 int
@@ -479,6 +507,8 @@ int
 glusterd_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
 int
 glusterd_sys_exec_output_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_snap_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
 int32_t
 glusterd_handle_node_rsp (dict_t *req_ctx, void *pending_entry,
                           glusterd_op_t op, dict_t *rsp_dict, dict_t *op_ctx,
@@ -504,7 +534,7 @@ glusterd_are_vol_all_peers_up (glusterd_volinfo_t *volinfo,
  * time a lock_owner is set
  */
 gf_boolean_t
-is_origin_glusterd ();
+is_origin_glusterd (dict_t *dict);
 
 gf_boolean_t
 glusterd_is_quorum_changed (dict_t *options, char *option, char *value);
@@ -565,4 +595,40 @@ glusterd_check_gsync_running_local (char *master, char *slave,
                                     char *conf_path,
                                     gf_boolean_t *is_run);
 
+gf_boolean_t
+glusterd_is_status_tasks_op (glusterd_op_t op, dict_t *dict);
+
+#ifdef GF_LINUX_HOST_OS
+char*
+glusterd_get_brick_mount_details (glusterd_brickinfo_t *brickinfo);
+struct mntent *
+glusterd_get_mnt_entry_info (char *mnt_pt, FILE *mtab);
+int
+glusterd_get_brick_root (char *path, char **mount_point);
+#endif //LINUX_HOST
+
+int
+glusterd_compare_snap_time(struct list_head *, struct list_head *);
+
+int
+glusterd_compare_snap_vol_time(struct list_head *, struct list_head *);
+
+int32_t
+glusterd_snap_volinfo_restore (dict_t *rsp_dict,
+                               glusterd_volinfo_t *new_volinfo,
+                               glusterd_volinfo_t *snap_volinfo);
+int32_t
+glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol);
+
+int32_t
+glusterd_missed_snapinfo_new (glusterd_missed_snap_info **missed_snapinfo);
+
+int32_t
+glusterd_missed_snap_op_new (glusterd_snap_op_t **snap_op);
+
+int32_t
+glusterd_add_missed_snaps_to_dict (dict_t *rsp_dict, char *snap_uuid,
+                                   glusterd_brickinfo_t *brickinfo,
+                                   int32_t brick_number, int32_t op);
+
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index fb91c3de4..6f3c69e7d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -28,6 +28,8 @@
 #include "logging.h"
 #include "dict.h"
 #include "graph-utils.h"
+#include "glusterd-store.h"
+#include "glusterd-hooks.h"
 #include "trie.h"
 #include "glusterd-mem-types.h"
 #include "cli1-xdr.h"
@@ -594,6 +596,8 @@ get_server_xlator (char *xlator)
                 subvol = GF_XLATOR_MARKER;
         if (strcmp (xlator, "io-stats") == 0)
                 subvol = GF_XLATOR_IO_STATS;
+        if (strcmp (xlator, "bd") == 0)
+                subvol = GF_XLATOR_BD;
 
         return subvol;
 }
@@ -1419,8 +1423,6 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         char     *password                = NULL;
         char     index_basepath[PATH_MAX] = {0};
         char     key[1024]                = {0};
-        char     *vgname                  = NULL;
-        char     *vg                      = NULL;
         glusterd_brickinfo_t *brickinfo   = NULL;
         char changelog_basepath[PATH_MAX] = {0,};
 
@@ -1441,47 +1443,43 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                 }
         }
 
-        if (volinfo->backend == GD_VOL_BK_BD) {
-                xl = volgen_graph_add (graph, "storage/bd_map", volname);
-                if (!xl)
-                        return -1;
+        xl = volgen_graph_add (graph, "storage/posix", volname);
+        if (!xl)
+                return -1;
 
-                ret = xlator_set_option (xl, "device", "vg");
-                if (ret)
-                        return -1;
+        ret = xlator_set_option (xl, "directory", path);
+        if (ret)
+                return -1;
 
-                vg = gf_strdup (path);
-                vgname = strrchr (vg, '/');
-                if (strchr(vg, '/') != vgname) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                  "invalid vg specified %s", path);
-                        GF_FREE (vg);
-                        goto out;
-                }
-                vgname++;
-                ret = xlator_set_option (xl, "export", vgname);
-                GF_FREE (vg);
-                if (ret)
-                        return -1;
-        } else {
-                xl = volgen_graph_add (graph, "storage/posix", volname);
+        ret = xlator_set_option (xl, "volume-id",
+                                 uuid_utoa (volinfo->volume_id));
+        if (ret)
+                return -1;
+
+        ret = check_and_add_debug_xl (graph, set_dict, volname,
+                                      "posix");
+        if (ret)
+                return -1;
+#ifdef HAVE_BD_XLATOR
+        if (*brickinfo->vg != '\0') {
+                /* Now add BD v2 xlator if volume is BD type */
+                xl = volgen_graph_add (graph, "storage/bd", volname);
                 if (!xl)
                         return -1;
 
-                ret = xlator_set_option (xl, "directory", path);
+                ret = xlator_set_option (xl, "device", "vg");
                 if (ret)
                         return -1;
-
-                ret = xlator_set_option (xl, "volume-id",
-                                 uuid_utoa (volinfo->volume_id));
+                ret = xlator_set_option (xl, "export", brickinfo->vg);
                 if (ret)
                         return -1;
 
-                ret = check_and_add_debug_xl (graph, set_dict, volname,
-                                                "posix");
+                ret = check_and_add_debug_xl (graph, set_dict, volname, "bd");
                 if (ret)
                         return -1;
+
         }
+#endif
 
         xl = volgen_graph_add (graph, "features/changelog", volname);
         if (!xl)
@@ -1616,7 +1614,8 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         }
 
         /* Check for read-only volume option, and add it to the graph */
-        if (dict_get_str_boolean (set_dict, "features.read-only", 0)) {
+        if (dict_get_str_boolean (set_dict, "features.read-only", 0)
+                || volinfo -> is_snap_volume) {
                 xl = volgen_graph_add (graph, "features/read-only", volname);
                 if (!xl) {
                         ret = -1;
@@ -1633,6 +1632,18 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                 }
         }
 
+        /* Check for compress volume option, and add it to the graph on server side */
+        if (dict_get_str_boolean (set_dict, "features.compress", 0)) {
+                xl = volgen_graph_add (graph, "features/cdc", volname);
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_str (set_dict, "compress.mode", "server");
+                if (ret)
+                        goto out;
+        }
+
         xl = volgen_graph_add_as (graph, "debug/io-stats", path);
         if (!xl)
                 return -1;
@@ -2444,6 +2455,29 @@ out:
         return ret;
 }
 
+static int client_graph_set_perf_options(volgen_graph_t *graph,
+					 glusterd_volinfo_t *volinfo,
+					 dict_t *set_dict)
+{
+	data_t *tmp_data = NULL;
+	char *volname = NULL;
+
+	/*
+	 * Logic to make sure NFS doesn't have performance translators by
+	 * default for a volume
+	 */
+	volname = volinfo->volname;
+	tmp_data = dict_get (set_dict, "nfs-volume-file");
+	if (!tmp_data)
+		return volgen_graph_set_options_generic(graph, set_dict,
+							volname,
+							&perfxl_option_handler);
+	else
+		return volgen_graph_set_options_generic(graph, set_dict,
+							volname,
+							&nfsperfxl_option_handler);
+}
+
 static int
 client_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                       dict_t *set_dict, void *param)
@@ -2451,7 +2485,6 @@ client_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         int       ret      = 0;
         xlator_t *xl       = NULL;
         char     *volname  = NULL;
-        data_t   *tmp_data = NULL;
 
         volname = volinfo->volname;
         ret = volgen_graph_build_clients (graph, volinfo, set_dict, param);
@@ -2462,6 +2495,31 @@ client_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         if (ret == -1)
                 goto out;
 
+        /* Check for compress volume option, and add it to the graph on client side */
+        if (dict_get_str_boolean (set_dict, "features.compress", 0)) {
+                xl = volgen_graph_add (graph, "features/cdc", volname);
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_str (set_dict, "compress.mode", "client");
+                if (ret)
+                        goto out;
+
+        }
+
+        ret = glusterd_volinfo_get_boolean (volinfo, "features.encryption");
+        if (ret == -1)
+                goto out;
+        if (ret) {
+                xl = volgen_graph_add (graph, "encryption/crypt", volname);
+
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
         ret = glusterd_volinfo_get_boolean (volinfo, VKEY_FEATURES_QUOTA);
         if (ret == -1)
                 goto out;
@@ -2487,16 +2545,7 @@ client_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                 }
         }
 
-        /* Logic to make sure NFS doesn't have performance translators by
-           default for a volume */
-        tmp_data = dict_get (set_dict, "nfs-volume-file");
-        if (!tmp_data)
-                ret = volgen_graph_set_options_generic (graph, set_dict, volinfo,
-                                                        &perfxl_option_handler);
-        else
-                ret = volgen_graph_set_options_generic (graph, set_dict, volname,
-                                                        &nfsperfxl_option_handler);
-
+	ret = client_graph_set_perf_options(graph, volinfo, set_dict);
         if (ret)
                 goto out;
 
@@ -3126,8 +3175,6 @@ glusterd_generate_brick_volfile (glusterd_volinfo_t *volinfo,
         return ret;
 }
 
-
-
 static void
 get_vol_tstamp_file (char *filename, glusterd_volinfo_t *volinfo)
 {
@@ -3394,6 +3441,54 @@ out:
 }
 
 int
+glusterd_check_nfs_topology_identical (gf_boolean_t *identical)
+{
+        char            nfsvol[PATH_MAX]        = {0,};
+        char            tmpnfsvol[PATH_MAX]     = {0,};
+        glusterd_conf_t *conf                   = NULL;
+        xlator_t        *this                   = THIS;
+        int             ret                     = -1;
+        int             tmpclean                = 0;
+        int             tmpfd                   = -1;
+
+        if ((!identical) || (!this) || (!this->private))
+                goto out;
+
+        conf = (glusterd_conf_t *) this->private;
+
+        /* Fetch the original NFS volfile */
+        glusterd_get_nodesvc_volfile ("nfs", conf->workdir,
+                                      nfsvol, sizeof (nfsvol));
+
+        /* Create the temporary NFS volfile */
+        snprintf (tmpnfsvol, sizeof (tmpnfsvol), "/tmp/gnfs-XXXXXX");
+        tmpfd = mkstemp (tmpnfsvol);
+        if (tmpfd < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                                    "Unable to create temp file %s: (%s)",
+                                    tmpnfsvol, strerror (errno));
+                goto out;
+        }
+
+        tmpclean = 1; /* SET the flag to unlink() tmpfile */
+
+        ret = glusterd_create_global_volfile (build_nfs_graph,
+                                              tmpnfsvol, NULL);
+        if (ret)
+                goto out;
+
+        /* Compare the topology of volfiles */
+        ret = glusterd_check_topology_identical (nfsvol, tmpnfsvol,
+                                                 identical);
+out:
+        if (tmpfd >= 0)
+                close (tmpfd);
+        if (tmpclean)
+                unlink (tmpnfsvol);
+        return ret;
+}
+
+int
 glusterd_check_nfs_volfile_identical (gf_boolean_t *identical)
 {
         char            nfsvol[PATH_MAX]        = {0,};
@@ -3842,3 +3937,129 @@ gd_is_boolean_option (char *key)
 
         return _gf_false;
 }
+
+/* This function will restore origin volume to it's snap.
+ * The restore operation will simply replace the Gluster origin
+ * volume with the snap volume.
+ * TODO: Multi-volume delete to be done.
+ *       Cleanup in case of restore failure is pending.
+ *
+ * @param orig_vol      volinfo of origin volume
+ * @param snap_vol      volinfo of snapshot volume
+ *
+ * @return 0 on success and negative value on error
+ */
+int
+gd_restore_snap_volume (dict_t *rsp_dict,
+                        glusterd_volinfo_t *orig_vol,
+                        glusterd_volinfo_t *snap_vol)
+{
+        int                     ret             = -1;
+        glusterd_volinfo_t      *new_volinfo    = NULL;
+        glusterd_snap_t         *snap           = NULL;
+        xlator_t                *this           = NULL;
+        glusterd_conf_t         *conf           = NULL;
+        glusterd_volinfo_t      *temp_volinfo   = NULL;
+        glusterd_volinfo_t      *voliter        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GF_VALIDATE_OR_GOTO (this->name, orig_vol, out);
+        GF_VALIDATE_OR_GOTO (this->name, snap_vol, out);
+        snap = snap_vol->snapshot;
+        GF_VALIDATE_OR_GOTO (this->name, snap, out);
+
+        /* Snap volume must be stoped before performing the
+         * restore operation.
+         */
+        ret = glusterd_stop_volume (snap_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to stop "
+                        "snap volume");
+                goto out;
+        }
+
+        /* Create a new volinfo for the restored volume */
+        ret = glusterd_volinfo_dup (snap_vol, &new_volinfo, _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create volinfo");
+                goto out;
+        }
+
+        /* Following entries need to be derived from origin volume. */
+        strcpy (new_volinfo->volname, orig_vol->volname);
+        uuid_copy (new_volinfo->volume_id, orig_vol->volume_id);
+        new_volinfo->snap_count = orig_vol->snap_count;
+        new_volinfo->snap_max_hard_limit = orig_vol->snap_max_hard_limit;
+        new_volinfo->is_volume_restored = _gf_true;
+
+        /* Bump the version of the restored volume, so that nodes *
+         * which are done can sync during handshake */
+        new_volinfo->version = orig_vol->version;
+
+        list_for_each_entry_safe (voliter, temp_volinfo,
+                         &orig_vol->snap_volumes, snapvol_list) {
+                list_add_tail (&voliter->snapvol_list,
+                               &new_volinfo->snap_volumes);
+        }
+        /* Copy the snap vol info to the new_volinfo.*/
+        ret = glusterd_snap_volinfo_restore (rsp_dict, new_volinfo, snap_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to restore snap");
+                (void)glusterd_volinfo_delete (new_volinfo);
+                goto out;
+        }
+
+        /* If the orig_vol is already restored then we should delete
+         * the backend LVMs */
+        if (orig_vol->is_volume_restored) {
+                ret = glusterd_lvm_snapshot_remove (rsp_dict, orig_vol);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to remove "
+                                "LVM backend");
+                        (void)glusterd_volinfo_delete (new_volinfo);
+                        goto out;
+                }
+        }
+
+        /* Once the new_volinfo is completely constructed then delete
+         * the orinal volinfo
+         */
+        ret = glusterd_volinfo_delete (orig_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to delete volinfo");
+                (void)glusterd_volinfo_delete (new_volinfo);
+                goto out;
+        }
+
+        /* New volinfo always shows the status as created. Therefore
+         * set the status to stop. */
+        glusterd_set_volume_status (new_volinfo, GLUSTERD_STATUS_STOPPED);
+
+        list_add_tail (&new_volinfo->vol_list, &conf->volumes);
+
+        /* Now delete the snap entry. As a first step delete the snap
+         * volume information stored in store. */
+        ret = glusterd_snap_remove (rsp_dict, snap, _gf_false, _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Failed to delete "
+                        "snap %s", snap->snapname);
+                goto out;
+        }
+
+        ret = glusterd_store_volinfo (new_volinfo,
+                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store volinfo");
+                goto out;
+        }
+
+        ret = 0;
+out:
+
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
index 0c1a76c12..fcbaaf93e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -75,6 +75,7 @@ typedef enum {
         GF_XLATOR_INDEX,
         GF_XLATOR_MARKER,
         GF_XLATOR_IO_STATS,
+        GF_XLATOR_BD,
         GF_XLATOR_NONE,
 } glusterd_server_xlator_t;
 
@@ -123,6 +124,10 @@ int glusterd_create_shd_volfile ();
 
 int glusterd_delete_volfile (glusterd_volinfo_t *volinfo,
                              glusterd_brickinfo_t *brickinfo);
+int
+glusterd_delete_snap_volfile (glusterd_volinfo_t *volinfo,
+                              glusterd_volinfo_t *snap_volinfo,
+                              glusterd_brickinfo_t *brickinfo);
 
 int glusterd_volinfo_get (glusterd_volinfo_t *volinfo, char *key, char **value);
 int glusterd_volinfo_get_boolean (glusterd_volinfo_t *volinfo, char *key);
@@ -136,13 +141,22 @@ glusterd_check_voloption_flags (char *key, int32_t flags);
 gf_boolean_t
 glusterd_is_valid_volfpath (char *volname, char *brick);
 int generate_brick_volfiles (glusterd_volinfo_t *volinfo);
+int generate_snap_brick_volfiles (glusterd_volinfo_t *volinfo,
+                                  glusterd_volinfo_t *snap_volinfo);
 int generate_client_volfiles (glusterd_volinfo_t *volinfo,
                               glusterd_client_type_t client_type);
+int
+generate_snap_client_volfiles (glusterd_volinfo_t *actual_volinfo,
+                               glusterd_volinfo_t *snap_volinfo,
+                               glusterd_client_type_t client_type,
+                               gf_boolean_t vol_restore);
 int glusterd_get_volopt_content (dict_t *dict, gf_boolean_t xml_out);
 char*
 glusterd_get_trans_type_rb (gf_transport_type ttype);
 int
 glusterd_check_nfs_volfile_identical (gf_boolean_t *identical);
+int
+glusterd_check_nfs_topology_identical (gf_boolean_t *identical);
 
 uint32_t
 glusterd_get_op_version_for_key (char *key);
@@ -156,4 +170,7 @@ gd_is_xlator_option (char *key);
 gf_boolean_t
 gd_is_boolean_option (char *key);
 
+int gd_restore_snap_volume (dict_t *rsp_dict,
+                            glusterd_volinfo_t *orig_vol,
+                            glusterd_volinfo_t *snap_vol);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index c1aea4136..0d322b9ad 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -30,6 +30,7 @@
 #define glusterd_op_start_volume_args_get(dict, volname, flags) \
         glusterd_op_stop_volume_args_get (dict, volname, flags)
 
+
 int
 __glusterd_handle_create_volume (rpcsvc_request_t *req)
 {
@@ -604,31 +605,96 @@ glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
 }
 
 #ifdef HAVE_BD_XLATOR
+/*
+ * Validates if given VG in the brick exists or not. Also checks if VG has
+ * GF_XATTR_VOL_ID_KEY tag set to avoid using same VG for multiple bricks.
+ * Tag is checked only during glusterd_op_stage_create_volume. Tag is set during
+ * glusterd_validate_and_create_brickpath().
+ * @brick - brick info, @check_tag - check for VG tag or not
+ * @msg - Error message to return to caller
+ */
 int
-glusterd_is_valid_vg (const char *name)
+glusterd_is_valid_vg (glusterd_brickinfo_t *brick, int check_tag, char *msg)
 {
-        lvm_t           handle   = NULL;
-        vg_t            vg       = NULL;
-        char            *vg_name = NULL;
-        int             retval   = -1;
+        lvm_t                     handle     = NULL;
+        vg_t                      vg         = NULL;
+        char                     *vg_name    = NULL;
+        int                       retval     = 0;
+        char                     *p          = NULL;
+        char                     *ptr        = NULL;
+        struct dm_list           *dm_lvlist = NULL;
+        struct dm_list           *dm_seglist = NULL;
+        struct lvm_lv_list       *lv_list    = NULL;
+        struct lvm_property_value prop       = {0, };
+        struct lvm_lvseg_list    *seglist    = NULL;
+        struct dm_list           *taglist    = NULL;
+        struct lvm_str_list      *strl       = NULL;
 
         handle = lvm_init (NULL);
         if (!handle) {
-                gf_log ("", GF_LOG_ERROR, "lvm_init failed");
+                sprintf (msg, "lvm_init failed, could not validate vg");
                 return -1;
         }
-        vg_name = gf_strdup (name);
-        vg = lvm_vg_open (handle, basename (vg_name), "r", 0);
+        if (*brick->vg == '\0') { /* BD xlator has vg in brick->path */
+                p = gf_strdup (brick->path);
+                vg_name = strtok_r (p, "/", &ptr);
+        } else
+                vg_name = brick->vg;
+
+        vg = lvm_vg_open (handle, vg_name, "r", 0);
         if (!vg) {
-                gf_log ("", GF_LOG_ERROR, "no such vg: %s", vg_name);
+                sprintf (msg, "no such vg: %s", vg_name);
+                retval = -1;
                 goto out;
         }
+        if (!check_tag)
+                goto next;
+
+        taglist = lvm_vg_get_tags (vg);
+        if (!taglist)
+                goto next;
+
+        dm_list_iterate_items (strl, taglist) {
+                if (!strncmp(strl->str, GF_XATTR_VOL_ID_KEY,
+                             strlen (GF_XATTR_VOL_ID_KEY))) {
+                            sprintf (msg, "VG %s is already part of"
+                                    " a brick", vg_name);
+                            retval = -1;
+                            goto out;
+                    }
+        }
+next:
+
+        brick->caps = CAPS_BD | CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
+
+        dm_lvlist = lvm_vg_list_lvs (vg);
+        if (!dm_lvlist)
+                goto out;
+
+        dm_list_iterate_items (lv_list, dm_lvlist) {
+                dm_seglist = lvm_lv_list_lvsegs (lv_list->lv);
+                dm_list_iterate_items (seglist, dm_seglist) {
+                        prop = lvm_lvseg_get_property (seglist->lvseg,
+                                                       "segtype");
+                        if (!prop.is_valid || !prop.value.string)
+                                continue;
+                        if (!strcmp (prop.value.string, "thin-pool")) {
+                                brick->caps |= CAPS_THIN;
+                                gf_log (THIS->name, GF_LOG_INFO, "Thin Pool "
+                                        "\"%s\" will be used for thin LVs",
+                                        lvm_lv_get_name (lv_list->lv));
+                                break;
+                        }
+                }
+        }
+
         retval = 0;
 out:
         if (vg)
                 lvm_vg_close (vg);
         lvm_quit (handle);
-        GF_FREE (vg_name);
+        if (p)
+                GF_FREE (p);
         return retval;
 }
 #endif
@@ -653,9 +719,6 @@ glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr)
         char                                    msg[2048] = {0};
         uuid_t                                  volume_uuid;
         char                                    *volume_uuid_str;
-#ifdef HAVE_BD_XLATOR
-        char                                    *dev_type = NULL;
-#endif
         gf_boolean_t                             is_force = _gf_false;
 
         this = THIS;
@@ -700,10 +763,6 @@ glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-#ifdef HAVE_BD_XLATOR
-        ret = dict_get_str (dict, "device", &dev_type);
-#endif
-
         ret = dict_get_str (dict, "bricks", &bricks);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Unable to get bricks for "
@@ -752,19 +811,15 @@ glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr)
                         goto out;
                 }
 
+                if (!uuid_compare (brick_info->uuid, MY_UUID)) {
+
 #ifdef HAVE_BD_XLATOR
-                if (dev_type) {
-                        ret = glusterd_is_valid_vg (brick_info->path);
-                        if (ret) {
-                                snprintf (msg, sizeof(msg), "invalid vg %s",
-                                  brick_info->path);
-                                goto out;
+                        if (brick_info->vg[0]) {
+                                ret = glusterd_is_valid_vg (brick_info, 1, msg);
+                                if (ret)
+                                        goto out;
                         }
-
-                        break;
-                } else
 #endif
-                if (!uuid_compare (brick_info->uuid, MY_UUID)) {
                         ret = glusterd_validate_and_create_brickpath (brick_info,
                                                           volume_uuid, op_errstr,
                                                           is_force);
@@ -862,6 +917,7 @@ glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr)
         uuid_t                                  volume_id = {0,};
         char                                    volid[50] = {0,};
         char                                    xattr_volid[50] = {0,};
+        int                                     caps = 0;
 
         this = THIS;
         GF_ASSERT (this);
@@ -911,9 +967,6 @@ glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr)
                 if (uuid_compare (brickinfo->uuid, MY_UUID))
                         continue;
 
-                if (volinfo->backend == GD_VOL_BK_BD)
-                        continue;
-
                 ret = gf_lstat_dir (brickinfo->path, NULL);
                 if (ret && (flags & GF_CLI_FLAG_OP_FORCE)) {
                         continue;
@@ -958,8 +1011,24 @@ glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr)
                         ret = -1;
                         goto out;
                 }
+#ifdef HAVE_BD_XLATOR
+                if (brickinfo->vg[0])
+                        caps = CAPS_BD | CAPS_THIN |
+                                CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
+                /* Check for VG/thin pool if its BD volume */
+                if (brickinfo->vg[0]) {
+                        ret = glusterd_is_valid_vg (brickinfo, 0, msg);
+                        if (ret)
+                                goto out;
+                        /* if anyone of the brick does not have thin support,
+                           disable it for entire volume */
+                        caps &= brickinfo->caps;
+                } else
+                        caps = 0;
+#endif
         }
 
+        volinfo->caps = caps;
         ret = 0;
 out:
         if (ret && (msg[0] != '\0')) {
@@ -1114,6 +1183,16 @@ glusterd_op_stage_delete_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        if (volinfo->snap_count > 0 || !list_empty(&volinfo->snap_volumes)) {
+                snprintf (msg, sizeof (msg), "Cannot delete Volume %s ,"
+                        "as it has %ld snapshots. "
+                        "To delete the volume, "
+                        "first delete all the snapshots under it.",
+                          volname, volinfo->snap_count);
+                ret = -1;
+                goto out;
+        }
+
         ret = 0;
 
 out:
@@ -1211,14 +1290,22 @@ glusterd_op_stage_heal_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        if ((heal_op != GF_AFR_OP_INDEX_SUMMARY) &&
-            !glusterd_is_nodesvc_online ("glustershd")) {
-                ret = -1;
-                *op_errstr = gf_strdup ("Self-heal daemon is not running."
-                                        " Check self-heal daemon log file.");
-                gf_log (this->name, GF_LOG_WARNING, "%s", "Self-heal daemon is "
-                        "not running. Check self-heal daemon log file.");
-                goto out;
+        switch (heal_op) {
+                case GF_AFR_OP_INDEX_SUMMARY:
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                        break;
+                default:
+                        if (!glusterd_is_nodesvc_online("glustershd")){
+                                ret = -1;
+                                *op_errstr = gf_strdup ("Self-heal daemon is "
+                                                "not running. Check self-heal "
+                                                "daemon log file.");
+                                gf_log (this->name, GF_LOG_WARNING, "%s",
+                                        "Self-heal daemon is not running."
+                                        "Check self-heal daemon log file.");
+                                goto out;
+                        }
         }
 
         ret = 0;
@@ -1342,109 +1429,6 @@ out:
         return ret;
 }
 
-#ifdef HAVE_BD_XLATOR
-int
-glusterd_op_stage_bd (dict_t *dict, char **op_errstr)
-{
-        int                     ret       = -1;
-        char                    *volname  = NULL;
-        char                    *path     = NULL;
-        char                    *size     = NULL;
-        glusterd_volinfo_t      *volinfo  = NULL;
-        char                    msg[2048] = {0,};
-        gf_xl_bd_op_t           bd_op     = GF_BD_OP_INVALID;
-        uint64_t                bytes     = 0;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (msg, sizeof(msg), "Failed to get volume name");
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "bd-op", (int32_t *)&bd_op);
-        if (ret) {
-                snprintf (msg, sizeof(msg), "Failed to get bd-op");
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "path", &path);
-        if (ret) {
-                snprintf (msg, sizeof(msg), "Failed to get path");
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        if (bd_op == GF_BD_OP_NEW_BD) {
-                ret = dict_get_str (dict, "size", &size);
-                if (ret) {
-                        snprintf (msg, sizeof(msg), "Failed to get size");
-                        gf_log ("", GF_LOG_ERROR, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-                        goto out;
-                }
-                if (gf_string2bytesize (size, &bytes) < 0) {
-                        snprintf (msg, sizeof(msg),
-                                  "Invalid size %s, suffix with KB, MB etc",
-                                   size);
-                        gf_log ("", GF_LOG_ERROR, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-                        ret = -1;
-                        goto out;
-                }
-        } else if (bd_op == GF_BD_OP_SNAPSHOT_BD) {
-                ret = dict_get_str (dict, "size", &size);
-                if (ret) {
-                        snprintf (msg, sizeof(msg), "Failed to get size");
-                        gf_log ("", GF_LOG_ERROR, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-                        goto out;
-                }
-
-                if (gf_string2bytesize (size, &bytes) < 0) {
-                        ret = -1;
-                        snprintf (msg, sizeof(msg),
-                                "Invalid size %s, suffix with KB, MB etc",
-                                size);
-                        gf_log ("", GF_LOG_ERROR, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-                        goto out;
-                }
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof(msg), "Volume %s does not exist",
-                          volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        ret = glusterd_validate_volume_id (dict, volinfo);
-        if (ret)
-                goto out;
-
-        if (!glusterd_is_volume_started (volinfo)) {
-                snprintf (msg, sizeof(msg), "Volume %s is not started",
-                          volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
-out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-#endif
-
 int
 glusterd_op_create_volume (dict_t *dict, char **op_errstr)
 {
@@ -1466,9 +1450,8 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
         char                 *str        = NULL;
         char                 *username   = NULL;
         char                 *password   = NULL;
-#ifdef HAVE_BD_XLATOR
-        char                 *device     = NULL;
-#endif
+        int                   caps       = 0;
+        char                  msg[1024] __attribute__((unused)) = {0, };
 
         this = THIS;
         GF_ASSERT (this);
@@ -1523,12 +1506,6 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-#ifdef HAVE_BD_XLATOR
-        ret = dict_get_str (dict, "device", &device);
-        if (!ret)
-                volinfo->backend = GD_VOL_BK_BD;
-#endif
-
         /* replica-count 1 means, no replication, file is in one brick only */
         volinfo->replica_count = 1;
         /* stripe-count 1 means, no striping, file is present as a whole */
@@ -1637,6 +1614,7 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
 
         if (count)
                 brick = strtok_r (brick_list+1, " \n", &saveptr);
+        caps = CAPS_BD | CAPS_THIN | CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
 
         while ( i <= count) {
                 ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
@@ -1649,6 +1627,27 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
                                 brickinfo->hostname, brickinfo->path);
                         goto out;
                 }
+
+#ifdef HAVE_BD_XLATOR
+                if (!uuid_compare (brickinfo->uuid, MY_UUID)) {
+                        if (brickinfo->vg[0]) {
+                                ret = glusterd_is_valid_vg (brickinfo, 0, msg);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR, "%s",
+                                                msg);
+                                        goto out;
+                                }
+
+                                /* if anyone of the brick does not have thin
+                                   support, disable it for entire volume */
+                                caps &= brickinfo->caps;
+
+
+                        } else
+                                caps = 0;
+                }
+#endif
+
                 list_add_tail (&brickinfo->brick_list, &volinfo->bricks);
                 brick = strtok_r (NULL, " \n", &saveptr);
                 i++;
@@ -1656,6 +1655,8 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
 
         gd_update_volume_op_versions (volinfo);
 
+        volinfo->caps = caps;
+
         ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
         if (ret) {
                 glusterd_store_delete_volume (volinfo);
@@ -1726,6 +1727,47 @@ out:
         return ret;
 }
 
+int
+glusterd_stop_volume (glusterd_volinfo_t *volinfo)
+{
+        int                     ret             = -1;
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+        xlator_t                *this           = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                ret = glusterd_brick_stop (volinfo, brickinfo, _gf_false);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to stop "
+                                "brick (%s)", brickinfo->path);
+                        goto out;
+                }
+        }
+
+        glusterd_set_volume_status (volinfo, GLUSTERD_STATUS_STOPPED);
+
+        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store volinfo of "
+                        "%s volume", volinfo->volname);
+                goto out;
+        }
+
+        ret = glusterd_nodesvcs_handle_graph_change (volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to notify graph "
+                        "change for %s volume", volinfo->volname);
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
 
 int
 glusterd_op_stop_volume (dict_t *dict)
@@ -1734,7 +1776,6 @@ glusterd_op_stop_volume (dict_t *dict)
         int                                     flags = 0;
         char                                    *volname = NULL;
         glusterd_volinfo_t                      *volinfo = NULL;
-        glusterd_brickinfo_t                    *brickinfo = NULL;
         xlator_t                                *this = NULL;
 
         this = THIS;
@@ -1751,19 +1792,12 @@ glusterd_op_stop_volume (dict_t *dict)
                 goto out;
         }
 
-        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = glusterd_brick_stop (volinfo, brickinfo, _gf_false);
-                if (ret)
-                        goto out;
-        }
-
-        glusterd_set_volume_status (volinfo, GLUSTERD_STATUS_STOPPED);
-
-        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret)
+        ret = glusterd_stop_volume (volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to stop %s volume",
+                        volname);
                 goto out;
-
-        ret = glusterd_nodesvcs_handle_graph_change (volinfo);
+        }
 out:
         return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 8c6c5b954..665a8b298 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -724,6 +724,34 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .flags      = OPT_FLAG_CLIENT_OPT
         },
 
+ 	/* Crypt xlator options */
+
+	{ .key         = "features.encryption",
+	  .voltype     = "encryption/crypt",
+	  .option      = "!feat",
+	  .value       = "off",
+	  .op_version  = 3,
+	  .description = "enable/disable client-side encryption for "
+                         "the volume.",
+	  .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+	},
+
+        { .key         = "encryption.master-key",
+          .voltype     = "encryption/crypt",
+          .op_version  = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "encryption.data-key-size",
+          .voltype     = "encryption/crypt",
+          .op_version  = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "encryption.block-size",
+          .voltype     = "encryption/crypt",
+          .op_version  = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
         /* Client xlator options */
         { .key        = "network.frame-timeout",
           .voltype    = "protocol/client",
@@ -808,6 +836,12 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .option      = "statedump-path",
           .op_version  = 1
         },
+        { .key         = "server.outstanding-rpc-limit",
+          .voltype     = "protocol/server",
+          .option      = "rpc.outstanding-rpc-limit",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
         { .key         = "features.lock-heal",
           .voltype     = "protocol/server",
           .option      = "lk-heal",
@@ -964,6 +998,50 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
         },
 
+#ifdef HAVE_LIB_Z
+        /* Compressor-decompressor xlator options
+         * defaults used from xlator/feature/compress/src/cdc.h
+         */
+        { .key         = "features.compress",
+          .voltype     = "features/cdc",
+          .option      = "!compress",
+          .value       = "off",
+          .type        = NO_DOC,
+          .op_version  = 2,
+          .description = "enable/disable compression translator"
+        },
+        { .key         = "compress.mode",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "compress.window-size",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "compress.mem-level",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "compress.min-size",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "compress.compression-level",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "compress.debug",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+ #endif
+
         /* Quota xlator options */
         { .key        = VKEY_FEATURES_LIMIT_USAGE,
           .voltype    = "features/quota",
@@ -1142,6 +1220,12 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .type        = GLOBAL_DOC,
           .op_version  = 1
         },
+        { .key         = "nfs.outstanding-rpc-limit",
+          .voltype     = "nfs/server",
+          .option      = "rpc.outstanding-rpc-limit",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
         { .key         = "nfs.port",
           .voltype     = "nfs/server",
           .option      = "nfs.port",
@@ -1248,6 +1332,24 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .type        = GLOBAL_DOC,
           .op_version  = 3
         },
+        { .key         = "nfs.read-size",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.read-size",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.write-size",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.write-size",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.readdir-size",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.readdir-size",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
 
         /* Other options which don't fit any place above */
         { .key        = "features.read-only",
@@ -1294,6 +1396,10 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .voltype     = "storage/posix",
           .op_version  = 3
         },
+        { .key         = "storage.bd-aio",
+          .voltype     = "storage/bd",
+          .op_version  = 3
+        },
         { .key        = "config.memory-accounting",
           .voltype    = "configuration",
           .option     = "!config",
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index dc35f8bf0..59288ada0 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2006-2013 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -36,6 +36,7 @@
 #include "glusterd-store.h"
 #include "glusterd-hooks.h"
 #include "glusterd-utils.h"
+#include "glusterd-locks.h"
 #include "common-utils.h"
 #include "run.h"
 
@@ -44,29 +45,43 @@
 #include "glusterd-mountbroker.h"
 
 extern struct rpcsvc_program gluster_handshake_prog;
+extern struct rpcsvc_program gluster_cli_getspec_prog;
 extern struct rpcsvc_program gluster_pmap_prog;
 extern glusterd_op_info_t opinfo;
 extern struct rpcsvc_program gd_svc_mgmt_prog;
+extern struct rpcsvc_program gd_svc_mgmt_v3_prog;
 extern struct rpcsvc_program gd_svc_peer_prog;
 extern struct rpcsvc_program gd_svc_cli_prog;
+extern struct rpcsvc_program gd_svc_cli_prog_ro;
 extern struct rpc_clnt_program gd_brick_prog;
 extern struct rpcsvc_program glusterd_mgmt_hndsk_prog;
 
+extern char snap_mount_folder[PATH_MAX];
+
 rpcsvc_cbk_program_t glusterd_cbk_prog = {
         .progname  = "Gluster Callback",
         .prognum   = GLUSTER_CBK_PROGRAM,
         .progver   = GLUSTER_CBK_VERSION,
 };
 
-struct rpcsvc_program *all_programs[] = {
+struct rpcsvc_program *gd_inet_programs[] = {
         &gd_svc_peer_prog,
-        &gd_svc_cli_prog,
+        &gd_svc_cli_prog_ro,
         &gd_svc_mgmt_prog,
+        &gd_svc_mgmt_v3_prog,
         &gluster_pmap_prog,
         &gluster_handshake_prog,
         &glusterd_mgmt_hndsk_prog,
 };
-int rpcsvc_programs_count = (sizeof (all_programs) / sizeof (all_programs[0]));
+int gd_inet_programs_count = (sizeof (gd_inet_programs) /
+                              sizeof (gd_inet_programs[0]));
+
+struct rpcsvc_program *gd_uds_programs[] = {
+        &gd_svc_cli_prog,
+        &gluster_cli_getspec_prog,
+};
+int gd_uds_programs_count = (sizeof (gd_uds_programs) /
+                             sizeof (gd_uds_programs[0]));
 
 const char *gd_op_list[GD_OP_MAX + 1] = {
         [GD_OP_NONE]                    = "Invalid op",
@@ -97,6 +112,7 @@ const char *gd_op_list[GD_OP_MAX + 1] = {
         [GD_OP_COPY_FILE]               = "Copy File",
         [GD_OP_SYS_EXEC]                = "Execute system commands",
         [GD_OP_GSYNC_CREATE]            = "Geo-replication Create",
+        [GD_OP_SNAP]                    = "Snapshot",
         [GD_OP_MAX]                     = "Invalid op"
 };
 
@@ -122,12 +138,12 @@ glusterd_uuid_init ()
         GF_ASSERT (this);
         priv = this->private;
 
-	ret = glusterd_retrieve_uuid ();
-	if (ret == 0) {
-		gf_log (this->name, GF_LOG_INFO,
-			"retrieved UUID: %s", uuid_utoa (priv->uuid));
-		return 0;
-	}
+        ret = glusterd_retrieve_uuid ();
+        if (ret == 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "retrieved UUID: %s", uuid_utoa (priv->uuid));
+                return 0;
+        }
 
         ret = glusterd_uuid_generate_save ();
 
@@ -937,6 +953,187 @@ glusterd_launch_synctask (synctask_fn_t fn, void *opaque)
                         " and other volume related services");
 }
 
+int
+glusterd_uds_rpcsvc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
+                            void *data)
+{
+        /* glusterd_rpcsvc_notify() does stuff that calls coming in from the
+         * unix domain socket don't need. This is just an empty function to be
+         * used for the uds listener. This will be used later if required.
+         */
+        return 0;
+}
+
+/* The glusterd unix domain socket listener only listens for cli */
+rpcsvc_t *
+glusterd_init_uds_listener (xlator_t *this)
+{
+        int             ret = -1;
+        dict_t          *options = NULL;
+        rpcsvc_t        *rpc = NULL;
+        data_t          *sock_data = NULL;
+        char            sockfile[PATH_MAX+1] = {0,};
+        int             i = 0;
+
+
+        GF_ASSERT (this);
+
+        sock_data = dict_get (this->options, "glusterd-sockfile");
+        if (!sock_data) {
+                strncpy (sockfile, DEFAULT_GLUSTERD_SOCKFILE, PATH_MAX);
+        } else {
+                strncpy (sockfile, sock_data->data, PATH_MAX);
+        }
+
+        options = dict_new ();
+        if (!options)
+                goto out;
+
+        ret = rpcsvc_transport_unix_options_build (&options, sockfile);
+        if (ret)
+                goto out;
+
+        rpc = rpcsvc_init (this, this->ctx, options, 8);
+        if (rpc == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = rpcsvc_register_notify (rpc, glusterd_uds_rpcsvc_notify,
+                                      this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Failed to register notify function");
+                goto out;
+        }
+
+        ret = rpcsvc_create_listeners (rpc, options, this->name);
+        if (ret != 1) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to create listener");
+                goto out;
+        }
+        ret = 0;
+
+        for (i = 0; i < gd_uds_programs_count; i++) {
+                ret = glusterd_program_register (this, rpc, gd_uds_programs[i]);
+                if (ret) {
+                        i--;
+                        for (; i >= 0; i--)
+                                rpcsvc_program_unregister (rpc,
+                                                           gd_uds_programs[i]);
+
+                        goto out;
+                }
+        }
+
+out:
+        if (options)
+                dict_unref (options);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to start glusterd "
+                        "unix domain socket listener.");
+                if (rpc) {
+                        GF_FREE (rpc);
+                        rpc = NULL;
+                }
+        }
+        return rpc;
+}
+
+void
+glusterd_stop_uds_listener (xlator_t *this)
+{
+        glusterd_conf_t         *conf = NULL;
+        rpcsvc_listener_t       *listener = NULL;
+        rpcsvc_listener_t       *next = NULL;
+
+        GF_ASSERT (this);
+        conf = this->private;
+
+        (void) rpcsvc_program_unregister (conf->uds_rpc, &gd_svc_cli_prog);
+        (void) rpcsvc_program_unregister (conf->uds_rpc, &gluster_handshake_prog);
+
+        list_for_each_entry_safe (listener, next, &conf->uds_rpc->listeners,
+                                  list) {
+                rpcsvc_listener_destroy (listener);
+        }
+
+        (void) rpcsvc_unregister_notify (conf->uds_rpc, glusterd_rpcsvc_notify,
+                                         this);
+
+        unlink (DEFAULT_GLUSTERD_SOCKFILE);
+
+        GF_FREE (conf->uds_rpc);
+        conf->uds_rpc = NULL;
+
+        return;
+}
+
+static int
+glusterd_init_snap_folder (xlator_t *this)
+{
+        int             ret = -1;
+        struct stat     buf = {0,};
+
+        GF_ASSERT (this);
+
+        /* Snapshot volumes are mounted under /var/run/gluster/snaps folder.
+         * But /var/run is normally a symbolic link to /run folder, which
+         * creates problems as the entry point in the mtab for the mount point
+         * and glusterd maintained entry point will be different. Therefore
+         * identify the correct run folder and use it for snap volume mounting.
+         */
+        ret = lstat (GLUSTERD_VAR_RUN_DIR, &buf);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "stat fails on %s, exiting. (errno = %d)",
+                        GLUSTERD_VAR_RUN_DIR, errno);
+                goto out;
+        }
+
+        /* If /var/run is symlink then use /run folder */
+        if (S_ISLNK (buf.st_mode)) {
+                strcpy (snap_mount_folder, GLUSTERD_RUN_DIR);
+        } else {
+                strcpy (snap_mount_folder, GLUSTERD_VAR_RUN_DIR);
+        }
+
+        strcat (snap_mount_folder, GLUSTERD_DEFAULT_SNAPS_BRICK_DIR);
+
+        ret = stat (snap_mount_folder, &buf);
+        if ((ret != 0) && (ENOENT != errno)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "stat fails on %s, exiting. (errno = %d)",
+                        snap_mount_folder, errno);
+                ret = -1;
+                goto out;
+        }
+
+        if ((!ret) && (!S_ISDIR(buf.st_mode))) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Provided snap path %s is not a directory,"
+                        "exiting", snap_mount_folder);
+                ret = -1;
+                goto out;
+        }
+
+        if ((-1 == ret) && (ENOENT == errno)) {
+                /* Create missing folders */
+                ret = mkdir_p (snap_mount_folder, 0777, _gf_false);
+
+                if (-1 == ret) {
+                        gf_log (this->name, GF_LOG_CRITICAL,
+                                "Unable to create directory %s"
+                                " ,errno = %d", snap_mount_folder, errno);
+                        goto out;
+                }
+        }
+
+out:
+        return ret;
+}
+
 /*
  * init - called during g
author	Krutika Dhananjay <kdhananj@redhat.com>	2015-07-23 14:59:19 +0530
committer	Pranith Kumar Karampuri <pkarampu@redhat.com>	2015-07-26 23:06:19 -0700
commit	fd296d90168002a475a9899a72a0dd1811f038ee (patch)
tree	5ca35dc8e73ea0b1722c82215c91a87e5fd70faf /doc/developer-guide
parent	7bec717f8850135368609fccf1b1c697af60c546 (diff)